Add tripleo_ha_wrapper role

This role is meant to help moving the HA pcs tasks to pure deployment
steps run on the host and not in containers any more. We tested this
ansible review with the THT patches at
https://review.opendev.org/#/q/status:open+project:openstack/tripleo-heat-templates+branch:master+topic:pcs_host3

Here is the testing protocol that was used in order to verify we did
not break anything:
1) Deployed overcloud + tempest
2) Ran a successful redeploy
2.1) Verified that a redeploy restarted containers globally when the
     configuration for the service changed
2.2) Verified that a redeploy did not restart containers which config
     did not change
2.3) Verified that the ordering of restarting bundles allows for
     brownfield deployments (i.e. first the bundle is restarted due
     to new bind-mounts and then the bundle is restarted with new
     configuration)
2.4) Verified that converge step after minor update is successful
3) Ran an initial successful minor update
3.1) Verified that the new image is indeed being run by pacemaker
     (i.e. :pcmklatest tag is applied to the new containers)
3.2) Verified that after converge no container is restarted
4) Re-run a subsequent successful noop minor update
4.1) Verify that containers are not restarted a second time after
     the cluster is stopped and started

Depends-On: I367cf4b65300be8dca0190b9adeab549018d4a56

Change-Id: Iaa7e89f0d25221c2a6ef0b81eb88a6f496f01696
Related-Bug: #1863442

doc/source/roles/role-tripleo_ha_wrappers.rst

@@ -0,0 +1,6 @@
+=========================
+Role - tripleo-ha-wrapper
+=========================
+
+.. ansibleautoplugin::
+  :role: tripleo_ansible/roles/tripleo_ha_wrapper

tripleo_ansible/roles/tripleo_ha_wrapper/defaults/main.yml

@@ -0,0 +1,26 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+# All variables intended for modification should place placed in this file.
+
+# All variables within this role should have a prefix of "tripleo_ha_wrapper"
+tripleo_ha_wrapper_debug: "{{ (ansible_verbosity | int) >= 2 | bool }}"
+tripleo_ha_wrapper_hide_sensitive_logs: true
+tripleo_ha_wrapper_config_basedir: "/var/lib/config-data/puppet-generated"
+tripleo_ha_wrapper_config_suffix: ".previous_run"
+tripleo_ha_wrapper_puppet_modulepath: "/etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules"
+tripleo_ha_wrapper_pcmk_restart_script: "/var/lib/container-config-scripts/pacemaker_restart_bundle.sh"

tripleo_ansible/roles/tripleo_ha_wrapper/meta/main.yml

@@ -0,0 +1,41 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+galaxy_info:
+  author: OpenStack
+  description: TripleO OpenStack Role -- tripleo_ha_wrapper
+  company: Red Hat
+  license: Apache-2.0
+  min_ansible_version: 2.7
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  platforms:
+    - name: CentOS
+      versions:
+        - 8
+
+  galaxy_tags:
+    - tripleo
+
+
+# List your role dependencies here, one per line. Be sure to remove the '[]' above,
+# if you add dependencies to this list.
+dependencies: []

tripleo_ansible/roles/tripleo_ha_wrapper/molecule/default/Dockerfile

@@ -0,0 +1,37 @@
+# Molecule managed
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

tripleo_ansible/roles/tripleo_ha_wrapper/molecule/default/converge.yml

@@ -0,0 +1,29 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  roles:
+    - role: "tripleo_ha_wrapper"
+  vars:
+    tripleo_ha_wrapper_service_name: "foo"
+    tripleo_ha_wrapper_resource_name: "foo"
+    tripleo_ha_wrapper_bundle_name: "foo-bundle"
+    tripleo_ha_wrapper_resource_state: "Master"
+    tripleo_ha_wrapper_puppet_execute: "notify{ foo: }"
+    tripleo_ha_wrapper_puppet_tags: "file"
+    tripleo_ha_wrapper_puppet_config_volume: "haproxy"

tripleo_ansible/roles/tripleo_ha_wrapper/molecule/default/molecule.yml

@@ -0,0 +1,38 @@
+---
+driver:
+  name: podman
+
+log: true
+
+platforms:
+  - name: centos8
+    hostname: centos8
+    image: centos:8
+    dockerfile: Dockerfile
+    pkg_extras: python*-setuptools
+    volumes:
+      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    ulimits: &ulimit
+      - host
+
+provisioner:
+  name: ansible
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - check
+    - verify
+    - destroy
+
+verifier:
+  name: testinfra

tripleo_ansible/roles/tripleo_ha_wrapper/molecule/default/prepare.yml

@@ -0,0 +1,53 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  roles:
+    - role: test_deps
+      vars:
+        test_deps_setup_tripleo: true
+
+  post_tasks:
+    - name: Create paths
+      file:
+        path: "{{ item }}"
+        state: directory
+        recurse: true
+      with_items:
+        - "/var/lib/container-config-scripts"
+        - "/var/lib/config-data/puppet-generated"
+
+    - name: Create haproxy md5sum
+      lineinfile:
+        path: "/var/lib/config-data/puppet-generated/haproxy.md5sum"
+        line: "faa59b504dcd9b2c7fb9b0ebf3569daa"
+        create: true
+
+    - name: Create fake puppet script
+      lineinfile:
+        path: "/usr/bin/puppet"
+        line: "#!/bin/bash"
+        create: true
+        mode: 0755
+
+    - name: Create bash script
+      lineinfile:
+        path: "/var/lib/container-config-scripts/pacemaker_restart_bundle.sh"
+        line: "#!/bin/bash"
+        create: true
+        mode: 0755

tripleo_ansible/roles/tripleo_ha_wrapper/molecule/default/verify.yml

@@ -0,0 +1,15 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.

tripleo_ansible/roles/tripleo_ha_wrapper/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the# License for the specific language governing permissions and limitations
+# under the License.
+#
+# This role is used to simplify the management of HA containers within TripleO
+# Specifically it does the following:
+# 1) It runs the so-called init bundles on the host which are in charge of creating the pcmk resources
+#    via puppet (or to tweak them in case they changed on a redeploy)
+# 2) They trigger calling the pacemaker_restart.sh script when the config for the HA service has changed
+#    This script will restart the HA resource globally from the bootstrap node in case the config changed there
+#    and it will only restart the service on the single node when the configuration changed and we are doing
+#    a minor update.
+
+# The following variables are required:
+# - tripleo_ha_wrapper_service_name: The name of the tripleo_service being used (e.g. mysql)
+# - tripleo_ha_wrapper_resource_name: The name of the ocf resource being used (e.g. galera)
+# - tripleo_ha_wrapper_bundle_name: The name of the pacemaker bundle being used (e.g. galera-bundle)
+# - tripleo_ha_wrapper_resource_state: The desired state of the resource (e.g. Master)
+# - tripleo_ha_wrapper_puppet_execute: 'include ::tripleo::....'
+# - tripleo_ha_wrapper_puppet_tags: 'pacemaker::resource::bundle,...'
+# - tripleo_ha_wrapper_puppet_config_volume: the folder name to lookd for md5 hashes
+# - tripleo_ha_wrapper_puppet_debug: Should puppet be run in debug mode (defaults to false)
+# - tripleo_ha_wrapper_minor_update: (true|'') is this a minor update workflow or not
+
+- name: Gather variables for each operating system
+  include_vars: "{{ item }}"
+  with_first_found:
+    - skip: true
+      files:
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
+        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+        - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+        - "{{ ansible_distribution | lower }}.yml"
+        - "{{ ansible_os_family | lower }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
+        - "{{ ansible_os_family | lower }}.yml"
+  tags:
+    - always
+
+- name: "Run init bundle puppet on the host for {{ tripleo_ha_wrapper_service_name }}"
+  shell: |
+    puppet apply {{ (tripleo_ha_wrapper_puppet_debug | default(false) | bool) | ternary('--debug --verbose', '') }} --detailed-exitcodes \
+    --summarize --color=false --modulepath '{{ tripleo_ha_wrapper_puppet_modulepath }}' --tags '{{ tripleo_ha_wrapper_puppet_tags }}' \
+    -e '{{ tripleo_ha_wrapper_puppet_execute }}'
+  register: puppet_run
+  changed_when: puppet_run.rc == 2
+  failed_when: puppet_run.rc != 2 and puppet_run.rc != 0
+
+- name: Run pacemaker restart if the config file for the service changed
+  tripleo_diff_exec:
+    command: >-
+      {{ tripleo_ha_wrapper_pcmk_restart_script }} {{ tripleo_ha_wrapper_service_name }}
+      {{ tripleo_ha_wrapper_resource_name }} {{ tripleo_ha_wrapper_bundle_name }}
+      {{ tripleo_ha_wrapper_resource_state }}
+    state_file: "{{ tripleo_ha_wrapper_config_basedir }}/{{ tripleo_ha_wrapper_puppet_config_volume }}.md5sum"
+    state_file_suffix: "{{ tripleo_ha_wrapper_config_suffix }}"
+    environment:
+      TRIPLEO_MINOR_UPDATE: "{{ tripleo_ha_wrapper_minor_update|default('') }}"

zuul.d/molecule.yaml

@@ -24,6 +24,7 @@
       - tripleo-ansible-centos-8-molecule-tripleo_create_admin
       - tripleo-ansible-centos-8-molecule-tripleo_derived_parameters
       - tripleo-ansible-centos-8-molecule-tripleo_firewall
+      - tripleo-ansible-centos-8-molecule-tripleo_ha_wrapper
       - tripleo-ansible-centos-8-molecule-tripleo_hieradata
       - tripleo-ansible-centos-8-molecule-tripleo_hosts_entries
       - tripleo-ansible-centos-8-molecule-tripleo_image_serve
@@ -73,6 +74,7 @@
       - tripleo-ansible-centos-8-molecule-tripleo_create_admin
       - tripleo-ansible-centos-8-molecule-tripleo_derived_parameters
       - tripleo-ansible-centos-8-molecule-tripleo_firewall
+      - tripleo-ansible-centos-8-molecule-tripleo_ha_wrapper
       - tripleo-ansible-centos-8-molecule-tripleo_hieradata
       - tripleo-ansible-centos-8-molecule-tripleo_hosts_entries
       - tripleo-ansible-centos-8-molecule-tripleo_image_serve
@@ -122,6 +124,7 @@
       - tripleo-ansible-centos-8-molecule-tripleo_container_tag
       - tripleo-ansible-centos-8-molecule-tripleo_create_admin
       - tripleo-ansible-centos-8-molecule-tripleo_firewall
+      - tripleo-ansible-centos-8-molecule-tripleo_ha_wrapper
       - tripleo-ansible-centos-8-molecule-tripleo_hieradata
       - tripleo-ansible-centos-8-molecule-tripleo_hosts_entries
       - tripleo-ansible-centos-8-molecule-tripleo_image_serve
@@ -302,6 +305,13 @@
     parent: tripleo-ansible-centos-8-base
     vars:
       tripleo_role_name: tripleo_firewall
+- job:
+    files:
+    - ^tripleo_ansible/roles/tripleo_ha_wrapper/.*
+    name: tripleo-ansible-centos-8-molecule-tripleo_ha_wrapper
+    parent: tripleo-ansible-centos-8-base
+    vars:
+      tripleo_role_name: tripleo_ha_wrapper
 - job:
     files:
     - ^tripleo_ansible/roles/tripleo_hieradata/.*