Fix update of pacemaker container images during major upgrade

Currently, the idiomatic "download image and retag to pcmklatest"
happens at step 2 during upgrade. This doesn't work if the stack
is already containerized before the upgrade, because pacemaker
is still running at step 2.

Reshuffle the steps at which the various upgrade tasks are run,
while keeping the ordering guarantees of the upgrade flow:

  . Deletion of non-containerized resources happens at step 1,
    to allow calling pcs while pacemaker is running.
  . Pacemaker is stopped at step 2.
  . Docker images for containerized resources are upgraded at
    step 3, after the cluster is guaranteed to be stopped.
  . Pacemaker is restarted at step 4 as before, once we know
    that all resources have been upgraded, yum packages updated
    and any potential docker restart has been executed.

Also change the way we detect containerized resources, so that
the predicate still remains valid past step 2 when pacemaker
has been stopped and has deleted its containerized resources.

Change-Id: I85e11dd93c7fd2c42e71b467f46b0044d4516524

docker/services/pacemaker/cinder-backup.yaml

@@ -243,19 +243,18 @@ outputs:
         - name: Get docker Cinder-Backup image
           set_fact:
             docker_image_latest: *cinder_backup_image_pcmklatest
-        - name: Check if Cinder-Backup is already containerized
-          shell: "docker ps -a | grep {{docker_image_latest}}"
+        - name: Check for Cinder-Backup Kolla configuration
+          command: grep '^backup_driver[ \t]*=' /var/lib/config-data/puppet-generated/cinder/etc/cinder/cinder.conf
+          changed_when: no
           ignore_errors: true
-          register: cinder_backup_containerized
-        - name: Retag the pacemaker image if containerized
-          when:
-            - step|int == 2
-            - cinder_backup_containerized|succeeded
-          block: *cinder_backup_fetch_retag_container_tasks
+          register: cinder_backup_kolla_config
+        - name: Check if Cinder-Backup is already containerized
+          set_fact:
+            cinder_backup_containerized: "{{cinder_backup_kolla_config|succeeded}}"
         - name: Cinder-Backup baremetal to container upgrade tasks
           when:
-            - step|int == 2
-            - cinder_backup_containerized|failed
+            - step|int == 1
+            - not cinder_backup_containerized|bool
           block:
             - name: get bootstrap nodeid
               tags: common
@@ -291,3 +290,8 @@ outputs:
                   until: output.rc == 0
             - name: Disable cinder_backup service
               service: name=openstack-cinder-backup enabled=no
+        - name: Retag the pacemaker image if containerized
+          when:
+            - step|int == 3
+            - cinder_backup_containerized|bool
+          block: *cinder_backup_fetch_retag_container_tasks

docker/services/pacemaker/cinder-volume.yaml

@@ -268,19 +268,18 @@ outputs:
         - name: Get docker Cinder-Volume image
           set_fact:
             docker_image_latest: *cinder_volume_image_pcmklatest
-        - name: Check if Cinder-Volume is already containerized
-          shell: "docker ps -a | grep {{docker_image_latest}}"
+        - name: Check for Cinder-Volume Kolla configuration
+          command: grep '^volume_driver[ \t]*=' /var/lib/config-data/puppet-generated/cinder/etc/cinder/cinder.conf
+          changed_when: no
           ignore_errors: true
-          register: cinder_volume_containerized
-        - name: Retag the pacemaker image if containerized
-          when:
-            - step|int == 2
-            - cinder_volume_containerized|succeeded
-          block: *cinder_volume_fetch_retag_container_tasks
+          register: cinder_volume_kolla_config
+        - name: Check if Cinder-Volume is already containerized
+          set_fact:
+            cinder_volume_containerized: "{{cinder_volume_kolla_config|succeeded}}"
         - name: Cinder-Volume baremetal to container upgrade tasks
           when:
-            - step|int == 2
-            - cinder_volume_containerized|failed
+            - step|int == 1
+            - not cinder_volume_containerized|bool
           block:
             - name: get bootstrap nodeid
               tags: common
@@ -316,6 +315,11 @@ outputs:
                   until: output.rc == 0
             - name: Disable cinder_volume service from boot
               service: name=openstack-cinder-volume enabled=no
+        - name: Retag the pacemaker image if containerized
+          when:
+            - step|int == 3
+            - cinder_volume_containerized|bool
+          block: *cinder_volume_fetch_retag_container_tasks
       fast_forward_upgrade_tasks:
         - name: Check cluster resource status
           pacemaker_resource:

docker/services/pacemaker/database/mysql.yaml

@@ -299,19 +299,17 @@ outputs:
         - name: Get docker Mysql image
           set_fact:
             docker_image_latest: *mysql_image_pcmklatest
+        - name: Check for Mysql Kolla configuration
+          stat:
+            path: /var/lib/config-data/puppet-generated/mysql
+          register: mysql_kolla_config
         - name: Check if Mysql is already containerized
-          shell: "docker ps -a | grep {{docker_image_latest}}"
-          ignore_errors: true
-          register: mysql_containerized
-        - name: Retag the pacemaker image if containerized
-          when:
-            - step|int == 2
-            - mysql_containerized|succeeded
-          block: *mysql_fetch_retag_container_tasks
+          set_fact:
+            mysql_containerized: "{{mysql_kolla_config.stat.isdir | default(false)}}"
         - name: Mysql baremetal to container upgrade tasks
           when:
-            - step|int == 2
-            - mysql_containerized|failed
+            - step|int == 1
+            - not mysql_containerized|bool
           block:
             - name: get bootstrap nodeid
               tags: common
@@ -351,3 +349,8 @@ outputs:
               file: state=absent path=/etc/xinetd.d/galera-monitor
             - name: Restart xinetd service after clustercheck removal
               service: name=xinetd state=restarted
+        - name: Retag the pacemaker image if containerized
+          when:
+            - step|int == 3
+            - mysql_containerized|bool
+          block: *mysql_fetch_retag_container_tasks

docker/services/pacemaker/database/redis.yaml

@@ -290,19 +290,17 @@ outputs:
         - name: Get docker redis image
           set_fact:
             docker_image_latest: *redis_image_pcmklatest
+        - name: Check for redis Kolla configuration
+          stat:
+            path: /var/lib/config-data/puppet-generated/redis
+          register: redis_kolla_config
         - name: Check if redis is already containerized
-          shell: "docker ps -a | grep {{docker_image_latest}}"
-          ignore_errors: true
-          register: redis_containerized
-        - name: Retag the pacemaker image if containerized
-          when:
-            - step|int == 2
-            - redis_containerized|succeeded
-          block: *redis_fetch_retag_container_tasks
+          set_fact:
+            redis_containerized: "{{redis_kolla_config.stat.isdir | default(false)}}"
         - name: redis baremetal to container upgrade tasks
           when:
-            - step|int == 2
-            - redis_containerized|failed
+            - step|int == 1
+            - not redis_containerized|bool
           block:
             - name: get bootstrap nodeid
               tags: common
@@ -338,6 +336,11 @@ outputs:
                   until: output.rc == 0
             - name: Disable redis service
               service: name=redis enabled=no
+        - name: Retag the pacemaker image if containerized
+          when:
+            - step|int == 3
+            - redis_containerized|bool
+          block: *redis_fetch_retag_container_tasks
       fast_forward_upgrade_tasks:
         - name: Check cluster resource status of redis
           pacemaker_resource:

docker/services/pacemaker/haproxy.yaml

@@ -251,19 +251,17 @@ outputs:
         - name: Get docker haproxy image
           set_fact:
             docker_image_latest: *haproxy_image_pcmklatest
+        - name: Check for haproxy Kolla configuration
+          stat:
+            path: /var/lib/config-data/puppet-generated/haproxy
+          register: haproxy_kolla_config
         - name: Check if haproxy is already containerized
-          shell: "docker ps -a | grep {{docker_image_latest}}"
-          ignore_errors: true
-          register: haproxy_containerized
-        - name: Retag the pacemaker image if containerized
-          when:
-            - step|int == 2
-            - haproxy_containerized|succeeded
-          block: *haproxy_fetch_retag_container_tasks
+          set_fact:
+            haproxy_containerized: "{{haproxy_kolla_config.stat.isdir | default(false)}}"
         - name: haproxy baremetal to container upgrade tasks
           when:
-            - step|int == 2
-            - haproxy_containerized|failed
+            - step|int == 1
+            - not haproxy_containerized|bool
           block:
             - name: get bootstrap nodeid
               tags: common
@@ -297,3 +295,8 @@ outputs:
                   register: output
                   retries: 5
                   until: output.rc == 0
+        - name: Retag the pacemaker image if containerized
+          when:
+            - step|int == 3
+            - haproxy_containerized|bool
+          block: *haproxy_fetch_retag_container_tasks

docker/services/pacemaker/manila-share.yaml

@@ -209,19 +209,22 @@ outputs:
         - name: Get docker Manila-Share image
           set_fact:
             docker_image_latest: *manila_share_image_pcmklatest
+        - name: Check for Manila-Share Kolla configuration
+          stat:
+            path: /var/lib/config-data/puppet-generated/manila
+          register: manila_share_kolla_config
         - name: Check if Manila-Share is already containerized
-          shell: "docker ps -a | grep {{docker_image_latest}}"
-          ignore_errors: true
-          register: manila_share_containerized
-        - name: Retag the pacemaker image if containerized
-          when:
-            - step|int == 2
-            - manila_share_containerized|succeeded
-          block: *manila_share_fetch_retag_container_tasks
+          set_fact:
+            manila_share_containerized: "{{manila_share_kolla_config.stat.isdir | default(false)}}"
         - name: Manila-Share baremetal to container upgrade tasks
           when:
-            - step|int == 2
-            - manila_share_containerized|failed
+            - step|int == 1
+            - not manila_share_containerized|bool
           block:
             - name: Stop and disable manila_share service
               service: name=openstack-manila-share state=stopped enabled=no
+        - name: Retag the pacemaker image if containerized
+          when:
+            - step|int == 3
+            - manila_share_containerized|bool
+          block: *manila_share_fetch_retag_container_tasks

docker/services/pacemaker/ovn-dbs.yaml

@@ -144,5 +144,5 @@ outputs:
           ignore_errors: true
       upgrade_tasks:
         - name: Stop and disable ovn-northd service
-          when: step|int == 2
+          when: step|int == 1
           service: name=ovn-northd state=stopped enabled=no

docker/services/pacemaker/rabbitmq.yaml

@@ -252,19 +252,17 @@ outputs:
         - name: Get docker Rabbitmq image
           set_fact:
             docker_image_latest: *rabbitmq_image_pcmklatest
+        - name: Check for Rabbitmq Kolla configuration
+          stat:
+            path: /var/lib/config-data/puppet-generated/rabbitmq
+          register: rabbit_kolla_config
         - name: Check if Rabbitmq is already containerized
-          shell: "docker ps -a | grep {{docker_image_latest}}"
-          ignore_errors: true
-          register: rabbit_containerized
-        - name: Retag the pacemaker image if containerized
-          when:
-            - step|int == 2
-            - rabbit_containerized|succeeded
-          block: *rabbitmq_fetch_retag_container_tasks
+          set_fact:
+            rabbit_containerized: "{{rabbit_kolla_config.stat.isdir | default(false)}}"
         - name: Rabbitmq baremetal to container upgrade tasks
           when:
-            - step|int == 2
-            - rabbit_containerized|failed
+            - step|int == 1
+            - not rabbit_containerized|bool
           block:
             - name: get bootstrap nodeid
               command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
@@ -298,6 +296,11 @@ outputs:
                   until: output.rc == 0
             - name: Disable rabbitmq service
               service: name=rabbitmq-server enabled=no
+        - name: Retag the pacemaker image if containerized
+          when:
+            - step|int == 3
+            - rabbit_containerized|bool
+          block: *rabbitmq_fetch_retag_container_tasks
       fast_forward_upgrade_tasks:
         - name: Check cluster resource status of rabbitmq
           pacemaker_resource:

puppet/services/pacemaker.yaml

@@ -154,7 +154,7 @@ outputs:
           async: 30
           poll: 4
         - name: Stop pacemaker cluster
-          when: step|int == 3
+          when: step|int == 2
           pacemaker_cluster: state=offline
         - name: Start pacemaker cluster
           when: step|int == 4