diff --git a/deployment/cinder/cinder-volume-pacemaker-puppet.yaml b/deployment/cinder/cinder-volume-pacemaker-puppet.yaml index f40f91c545..43fa10313f 100644 --- a/deployment/cinder/cinder-volume-pacemaker-puppet.yaml +++ b/deployment/cinder/cinder-volume-pacemaker-puppet.yaml @@ -230,65 +230,74 @@ outputs: vars: container_image: {get_param: DockerCinderVolumeImage} container_image_latest: *cinder_volume_image_pcmklatest + update_tasks: - - name: Cinder-Volume fetch and retag container image for pacemaker + - name: cinder_volume fetch and retag container image for pacemaker when: step|int == 2 block: &cinder_volume_fetch_retag_container_tasks - - name: Get docker Cinder-Volume image + - name: Get container cinder_volume image set_fact: - docker_image: {get_param: DockerCinderVolumeImage} - docker_image_latest: *cinder_volume_image_pcmklatest - - name: Get previous Cinder-Volume image id - shell: "{{container_cli}} images | awk '/cinder-volume.* pcmklatest/{print $3}' | uniq" - register: cinder_volume_image_id - - block: - - name: Get a list of container using Cinder-Volume image - shell: "{{container_cli}} ps -a -q -f 'ancestor={{cinder_volume_image_id.stdout}}'" - register: cinder_volume_containers_to_destroy - # It will be recreated with the delpoy step. - - name: Remove any container using the same Cinder-Volume image - shell: "{{container_cli}} rm -fv {{item}}" - with_items: "{{ cinder_volume_containers_to_destroy.stdout_lines }}" - - name: Remove previous Cinder-Volume images - shell: "{{container_cli}} rmi -f {{cinder_volume_image_id.stdout}}" - when: - - cinder_volume_image_id.stdout != '' - - name: Pull latest Cinder-Volume images - command: "{{container_cli}} pull {{docker_image}}" - - name: Retag pcmklatest to latest Cinder-Volume image - import_role: + cinder_volume_image: {get_param: DockerCinderVolumeImage} + cinder_volume_image_latest: *cinder_volume_image_pcmklatest + - name: Pull latest cinder_volume images + command: "{{container_cli}} pull {{cinder_volume_image}}" + - name: Get previous cinder_volume image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{cinder_volume_image_latest}}" + register: old_cinder_volume_image_id + failed_when: false + - name: Get new cinder_volume image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{cinder_volume_image}}" + register: new_cinder_volume_image_id + - name: Retag pcmklatest to latest cinder_volume image + include_role: name: tripleo-container-tag vars: - container_image: "{{docker_image}}" - container_image_latest: "{{docker_image_latest}}" - # Got to check that pacemaker_is_active is working fine with bundle. - # TODO: pacemaker_is_active resource doesn't support bundle. + container_image: "{{cinder_volume_image}}" + container_image_latest: "{{cinder_volume_image_latest}}" + when: + - old_cinder_volume_image_id.stdout != new_cinder_volume_image_id.stdout + - block: + - name: Get a list of container using cinder_volume image + shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_cinder_volume_image_id.stdout}}'" + register: cinder_volume_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same cinder_volume image + shell: "{{container_cli}} rm -fv {{item}}" + with_items: "{{ cinder_volume_containers_to_destroy.stdout_lines }}" + - name: Remove previous cinder_volume images + shell: "{{container_cli}} rmi -f {{old_cinder_volume_image_id.stdout}}" + when: + - old_cinder_volume_image_id.stdout != '' + - old_cinder_volume_image_id.stdout != new_cinder_volume_image_id.stdout + upgrade_tasks: - - when: step|int == 0 - tags: common + - name: Prepare switch of cinder_volume image name + when: + - step|int == 0 block: - - name: Get docker Cinder-Volume image + - name: Get cinder_volume image id currently used by pacemaker + shell: "pcs resource config openstack-cinder-volume | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'" + register: cinder_volume_image_current_res + failed_when: false + - name: cinder_volume image facts set_fact: - cinder_volume_docker_image_latest: *cinder_volume_image_pcmklatest + cinder_volume_image_latest: *cinder_volume_image_pcmklatest + cinder_volume_image_current: "{{cinder_volume_image_current_res.stdout}}" - name: Prepare the switch to new cinder_volume container image name in pacemaker block: - - name: Get cinder_volume image id currently used by pacemaker - shell: "{{container_cli}} images | awk '/cinder-volume.* pcmklatest/{print $3}' | uniq" - register: cinder_volume_current_pcmklatest_id - name: Temporarily tag the current cinder_volume image id with the upgraded image name import_role: name: tripleo-container-tag vars: - container_image: "{{cinder_volume_current_pcmklatest_id.stdout}}" - container_image_latest: "{{cinder_volume_docker_image_latest}}" + container_image: "{{cinder_volume_image_current}}" + container_image_latest: "{{cinder_volume_image_latest}}" pull_image: false - when: cinder_volume_current_pcmklatest_id.stdout != '' + when: + - cinder_volume_image_current != '' + - cinder_volume_image_current != cinder_volume_image_latest - name: Check openstack-cinder-volume cluster resource status - pacemaker_resource: - resource: openstack-cinder-volume - state: show - check_mode: false - ignore_errors: true + shell: pcs resource config openstack-cinder-volume + failed_when: false register: cinder_volume_pcs_res_result - name: Set fact cinder_volume_pcs_res set_fact: @@ -301,6 +310,7 @@ outputs: - step|int == 1 - is_cinder_volume_bootstrap_node - cinder_volume_pcs_res|bool + - cinder_volume_image_current != cinder_volume_image_latest block: - name: Disable the cinder_volume cluster resource before container upgrade pacemaker_resource: @@ -311,7 +321,7 @@ outputs: retries: 5 until: output.rc == 0 - name: pcs resource bundle update cinder_volume for new container image name - command: "pcs resource bundle update openstack-cinder-volume container image={{cinder_volume_docker_image_latest}}" + command: "pcs resource bundle update openstack-cinder-volume container image={{cinder_volume_image_latest}}" - name: Enable the cinder_volume cluster resource when: pacemaker_resource: @@ -325,12 +335,14 @@ outputs: when: - step|int == 3 block: *cinder_volume_fetch_retag_container_tasks + post_upgrade_tasks: - name: Start cinder_volume service (pacemaker) when: step|int == 1 pacemaker_resource: resource: openstack-cinder-volume state: enable + fast_forward_upgrade_tasks: - when: - step|int == 0 diff --git a/deployment/database/mysql-pacemaker-puppet.yaml b/deployment/database/mysql-pacemaker-puppet.yaml index b8d6176ec3..82de7bc5c3 100644 --- a/deployment/database/mysql-pacemaker-puppet.yaml +++ b/deployment/database/mysql-pacemaker-puppet.yaml @@ -349,39 +349,46 @@ outputs: vars: container_image: {get_param: DockerMysqlImage} container_image_latest: *mysql_image_pcmklatest + update_tasks: - name: Mariadb fetch and retag container image for pacemaker when: step|int == 2 block: &mysql_fetch_retag_container_tasks - - name: Get docker Mariadb image + - name: Get container galera image set_fact: - docker_image: {get_param: DockerMysqlImage} - docker_image_latest: *mysql_image_pcmklatest - - name: Get previous Mariadb image id - shell: "{{container_cli}} images | awk '/mariadb.* pcmklatest/{print $3}' | uniq" - register: mariadb_image_id - - block: - - name: Get a list of container using Mariadb image - shell: "{{container_cli}} ps -a -q -f 'ancestor={{mariadb_image_id.stdout}}'" - register: mariadb_containers_to_destroy - # It will be recreated with the delpoy step. - - name: Remove any container using the same Mariadb image - shell: "{{container_cli}} rm -fv {{item}}" - with_items: "{{ mariadb_containers_to_destroy.stdout_lines }}" - - name: Remove previous Mariadb images - shell: "{{container_cli}} rmi -f {{mariadb_image_id.stdout}}" - when: - - mariadb_image_id.stdout != '' - - name: Pull latest Mariadb images - command: "{{container_cli}} pull {{docker_image}}" - - name: Retag pcmklatest to latest Mariadb image - import_role: + galera_image: {get_param: DockerMysqlImage} + galera_image_latest: *mysql_image_pcmklatest + - name: Pull latest galera images + command: "{{container_cli}} pull {{galera_image}}" + - name: Get previous galera image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{galera_image_latest}}" + register: old_galera_image_id + failed_when: false + - name: Get new galera image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{galera_image}}" + register: new_galera_image_id + - name: Retag pcmklatest to latest galera image + include_role: name: tripleo-container-tag vars: - container_image: "{{docker_image}}" - container_image_latest: "{{docker_image_latest}}" - # Got to check that pacemaker_is_active is working fine with bundle. - # TODO: pacemaker_is_active resource doesn't support bundle. + container_image: "{{galera_image}}" + container_image_latest: "{{galera_image_latest}}" + when: + - old_galera_image_id.stdout != new_galera_image_id.stdout + - block: + - name: Get a list of container using galera image + shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_galera_image_id.stdout}}'" + register: galera_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same galera image + shell: "{{container_cli}} rm -fv {{item}}" + with_items: "{{ galera_containers_to_destroy.stdout_lines }}" + - name: Remove previous galera images + shell: "{{container_cli}} rmi -f {{old_galera_image_id.stdout}}" + when: + - old_galera_image_id.stdout != '' + - old_galera_image_id.stdout != new_galera_image_id.stdout + upgrade_tasks: - vars: mysql_upgrade_persist: {get_param: MysqlUpgradePersist} @@ -415,6 +422,80 @@ outputs: tasks_from: restore.yml vars: tripleo_persist_dir: /var/lib/mysql + + - name: Prepare switch of galera image name + when: + - step|int == 0 + block: + - name: Get galera image id currently used by pacemaker + shell: "pcs resource config galera-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'" + register: galera_image_current_res + failed_when: false + - name: Image facts for galera + set_fact: + galera_image_latest: *mysql_image_pcmklatest + galera_image_current: "{{galera_image_current_res.stdout}}" + - name: Prepare the switch to new galera container image name in pacemaker + block: + - name: Temporarily tag the current galera image id with the upgraded image name + import_role: + name: tripleo-container-tag + vars: + container_image: "{{galera_image_current}}" + container_image_latest: "{{galera_image_latest}}" + pull_image: false + when: + - galera_image_current != '' + - galera_image_current != galera_image_latest + - name: Check galera cluster resource status + shell: pcs resource config galera-bundle + failed_when: false + register: galera_pcs_res_result + - name: Set fact galera_pcs_res + set_fact: + galera_pcs_res: "{{galera_pcs_res_result|succeeded}}" + - name: set is_mysql_bootstrap_node fact + tags: common + set_fact: is_mysql_bootstrap_node={{mysql_short_bootstrap_node_name|lower == ansible_hostname|lower}} + - name: Update galera pcs resource bundle for new container image + when: + - step|int == 1 + - is_mysql_bootstrap_node|bool + - galera_pcs_res|bool + - galera_image_current != galera_image_latest + block: + - name: Disable the galera cluster resource before container upgrade + pacemaker_resource: + resource: galera + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + - name: Move Mysql logging to /var/log/containers + block: + - name: Check Mysql logging configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='mysql-log']" + ignore_errors: true + register: mysql_logs_moved + - name: Change Mysql logging configuration in pacemaker + # rc == 6 means the configuration doesn't exist in the CIB + when: mysql_logs_moved.rc == 6 + block: + - name: Add a bind mount for logging in the galera bundle + command: pcs resource bundle update galera-bundle storage-map add id=mysql-log source-dir=/var/log/containers/mysql target-dir=/var/log/mysql options=rw + - name: Reconfigure Mysql log file in the galera resource agent + command: pcs resource update galera log=/var/log/mysql/mysqld.log + - name: Update the galera bundle to use the new container image name + command: "pcs resource bundle update galera-bundle container image={{galera_image_latest}}" + - name: Enable the galera cluster resource + pacemaker_resource: + resource: galera + state: enable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 - name: Create hiera data to upgrade mysql in a stepwise manner. when: - step|int == 1 @@ -471,77 +552,6 @@ outputs: - mysql_short_node_names_override - mysql_node_names_override when: mysql_short_node_names_upgraded | length == mysql_node_names | length - # Manage image update for pacemaker managed container. - - name: mysql baremetal to container upgrade tasks. - when: step|int == 0 - tags: common - block: - - name: Get docker Mysql image - set_fact: - mysql_docker_image_latest: *mysql_image_pcmklatest - - name: set is_mysql_bootstrap_node fact - set_fact: is_mysql_bootstrap_node={{mysql_short_bootstrap_node_name|lower == ansible_hostname|lower}} - - name: Prepare the switch to new galera container image name in pacemaker - block: - - name: Get galera image id currently used by pacemaker - shell: "{{container_cli}} images | awk '/mariadb.* pcmklatest/{print $3}' | uniq" - register: galera_current_pcmklatest_id - - name: Temporarily tag the current galera image id with the upgraded image name - import_role: - name: tripleo-container-tag - vars: - container_image: "{{galera_current_pcmklatest_id.stdout}}" - container_image_latest: "{{mysql_docker_image_latest}}" - pull_image: false - when: galera_current_pcmklatest_id.stdout != '' - - name: Check galera cluster resource status - pacemaker_resource: - resource: galera - state: show - check_mode: false - ignore_errors: true - register: galera_pcs_res_result - - name: Set fact galera_pcs_res - set_fact: - galera_pcs_res: "{{galera_pcs_res_result|succeeded}}" - - name: Update galera pcs resource bundle for new container image - when: - - step|int == 1 - - is_mysql_bootstrap_node - - galera_pcs_res|bool - block: - - name: Disable the galera cluster resource before container upgrade - pacemaker_resource: - resource: galera - state: disable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - - name: Move Mysql logging to /var/log/containers - block: - - name: Check Mysql logging configuration in pacemaker - command: cibadmin --query --xpath "//storage-mapping[@id='mysql-log']" - ignore_errors: true - register: mysql_logs_moved - - name: Change Mysql logging configuration in pacemaker - # rc == 6 means the configuration doesn't exist in the CIB - when: mysql_logs_moved.rc == 6 - block: - - name: Add a bind mount for logging in the galera bundle - command: pcs resource bundle update galera-bundle storage-map add id=mysql-log source-dir=/var/log/containers/mysql target-dir=/var/log/mysql options=rw - - name: Reconfigure Mysql log file in the galera resource agent - command: pcs resource update galera log=/var/log/mysql/mysqld.log - - name: Update the galera bundle to use the new container image name - command: "pcs resource bundle update galera-bundle container image={{mysql_docker_image_latest}}" - - name: Enable the galera cluster resource - pacemaker_resource: - resource: galera - state: enable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - name: Retag the pacemaker image if containerized when: - step|int == 3 @@ -591,6 +601,7 @@ outputs: UPGRADE_VOLUMES: "-v {{ mysql_upgrade_db_bind_mounts | union(['/tmp/mariadb-upgrade:/var/log/mariadb:rw']) | join(' -v ')}}" UPGRADE_SCRIPT: "{{mysql_upgrade_script}}" CONTAINER_CLI: "{{ container_cli }}" + external_upgrade_tasks: - vars: mysql_upgrade_transfer: {get_param: MysqlUpgradeTransfer} @@ -609,6 +620,7 @@ outputs: tripleo_transfer_src_host: "{{hostvars[groups['overcloud'][0]]['mysql_short_node_names'][1]}}" tripleo_transfer_dest_dir: /var/lib/mysql tripleo_transfer_dest_host: "{{hostvars[groups['overcloud'][0]]['mysql_short_bootstrap_node_name']}}" + fast_forward_upgrade_tasks: - when: - step|int == 6 diff --git a/deployment/database/redis-pacemaker-puppet.yaml b/deployment/database/redis-pacemaker-puppet.yaml index a8ee28c888..9cfdbb74d1 100644 --- a/deployment/database/redis-pacemaker-puppet.yaml +++ b/deployment/database/redis-pacemaker-puppet.yaml @@ -325,36 +325,105 @@ outputs: - name: Redis fetch and retag container image for pacemaker when: step|int == 2 block: &redis_fetch_retag_container_tasks - - name: Get docker Redis image + - name: Get container redis image set_fact: - docker_image: {get_param: DockerRedisImage} - docker_image_latest: *redis_image_pcmklatest - - name: Get previous Redis image id - shell: "{{container_cli}} images | awk '/redis.* pcmklatest/{print $3}' | uniq" - register: redis_image_id - - block: - - name: Get a list of container using Redis image - shell: "{{container_cli}} ps -a -q -f 'ancestor={{redis_image_id.stdout}}'" - register: redis_containers_to_destroy - # It will be recreated with the delpoy step. - - name: Remove any container using the same Redis image - shell: "{{container_cli}} rm -fv {{item}}" - with_items: "{{ redis_containers_to_destroy.stdout_lines }}" - - name: Remove previous Redis images - shell: "{{container_cli}} rmi -f {{redis_image_id.stdout}}" - when: - - redis_image_id.stdout != '' - - name: Pull latest Redis images - command: "{{container_cli}} pull {{docker_image}}" - - name: Retag pcmklatest to latest Redis image - import_role: + redis_image: {get_param: DockerRedisImage} + redis_image_latest: *redis_image_pcmklatest + - name: Pull latest redis images + command: "{{container_cli}} pull {{redis_image}}" + - name: Get previous redis image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{redis_image_latest}}" + register: old_redis_image_id + failed_when: false + - name: Get new redis image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{redis_image}}" + register: new_redis_image_id + - name: Retag pcmklatest to latest redis image + include_role: name: tripleo-container-tag vars: - container_image: "{{docker_image}}" - container_image_latest: "{{docker_image_latest}}" - # Got to check that pacemaker_is_active is working fine with bundle. - # TODO: pacemaker_is_active resource doesn't support bundle. + container_image: "{{redis_image}}" + container_image_latest: "{{redis_image_latest}}" + when: + - old_redis_image_id.stdout != new_redis_image_id.stdout + - block: + - name: Get a list of container using redis image + shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_redis_image_id.stdout}}'" + register: redis_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same redis image + shell: "{{container_cli}} rm -fv {{item}}" + with_items: "{{ redis_containers_to_destroy.stdout_lines }}" + - name: Remove previous redis images + shell: "{{container_cli}} rmi -f {{old_redis_image_id.stdout}}" + when: + - old_redis_image_id.stdout != '' + - old_redis_image_id.stdout != new_redis_image_id.stdout + upgrade_tasks: + - name: Prepare switch of redis image name + when: + - step|int == 0 + block: + - name: Get redis image id currently used by pacemaker + shell: "pcs resource config redis-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'" + register: redis_image_current_res + failed_when: false + - name: Image facts for redis + set_fact: + redis_image_latest: *redis_image_pcmklatest + redis_image_current: "{{redis_image_current_res.stdout}}" + - name: Prepare the switch to new redis container image name in pacemaker + block: + - name: Temporarily tag the current redis image id with the upgraded image name + import_role: + name: tripleo-container-tag + vars: + container_image: "{{redis_image_current}}" + container_image_latest: "{{redis_image_latest}}" + pull_image: false + when: + - redis_image_current != '' + - redis_image_current != redis_image_latest + - name: Check redis cluster resource status + shell: pcs resource config redis-bundle + failed_when: false + register: redis_pcs_res_result + - name: Set upgrade redis facts + set_fact: + redis_pcs_res: "{{redis_pcs_res_result|succeeded}}" + is_redis_bootstrap_node: "{{redis_short_bootstrap_node_name|lower == ansible_hostname|lower}}" + - name: Update redis-bundle pcs resource bundle for new container image + when: + - step|int == 1 + - is_redis_bootstrap_node|bool + - redis_pcs_res|bool + - redis_image_current != redis_image_latest + block: + - name: Disable the redis cluster resource before container upgrade + pacemaker_resource: + resource: redis-bundle + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + - name: Move redis logging to /var/log/containers + block: + - name: Check redis logging configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='redis-log' and @source-dir='/var/log/containers/redis']" + ignore_errors: true + register: redis_logs_moved + - name: Change redis logging configuration in pacemaker + # rc == 6 means the configuration doesn't exist in the CIB + when: redis_logs_moved.rc == 6 + block: + - name: Remove old bind mount for logging in the redis bundle + command: pcs resource bundle update redis-bundle storage-map remove redis-log + - name: Add a bind mount for logging in the redis bundle + command: pcs resource bundle update redis-bundle storage-map add id=redis-log source-dir=/var/log/containers/redis target-dir=/var/log/redis options=rw + - name: Update the redis bundle to use the new container image name + command: "pcs resource bundle update redis-bundle container image={{redis_image_latest}}" - when: - step|int == 1 block: @@ -397,71 +466,11 @@ outputs: vars: tripleo_upgrade_key: redis_short_node_names_override when: redis_short_node_names_upgraded | length == redis_short_node_names | length - - when: step|int == 0 - tags: common - block: - - name: Get docker redis image - set_fact: - redis_docker_image_latest: *redis_image_pcmklatest - - name: set is_redis_bootstrap_node fact - set_fact: is_redis_bootstrap_node={{redis_short_bootstrap_node_name|lower == ansible_hostname|lower}} - - name: Prepare the switch to new redis container image name in pacemaker - block: - - name: Get redis image id currently used by pacemaker - shell: "{{container_cli}} images | awk '/redis.* pcmklatest/{print $3}' | uniq" - register: redis_current_pcmklatest_id - - name: Temporarily tag the current redis image id with the upgraded image name - import_role: - name: tripleo-container-tag - vars: - container_image: "{{redis_current_pcmklatest_id.stdout}}" - container_image_latest: "{{redis_docker_image_latest}}" - pull_image: false - when: redis_current_pcmklatest_id.stdout != '' - - name: Check redis-bundle cluster resource status - pacemaker_resource: - resource: redis-bundle - state: show - check_mode: false - ignore_errors: true - register: redis_pcs_res_result - - name: Set fact redis_pcs_res - set_fact: - redis_pcs_res: "{{redis_pcs_res_result|succeeded}}" - - name: Update redis-bundle pcs resource bundle for new container image - when: - - step|int == 1 - - is_redis_bootstrap_node - - redis_pcs_res|bool - block: - - name: Disable the redis cluster resource before container upgrade - pacemaker_resource: - resource: redis-bundle - state: disable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - - name: Move redis logging to /var/log/containers - block: - - name: Check redis logging configuration in pacemaker - command: cibadmin --query --xpath "//storage-mapping[@id='redis-log' and @source-dir='/var/log/containers/redis']" - ignore_errors: true - register: redis_logs_moved - - name: Change redis logging configuration in pacemaker - # rc == 6 means the configuration doesn't exist in the CIB - when: redis_logs_moved.rc == 6 - block: - - name: Remove old bind mount for logging in the redis bundle - command: pcs resource bundle update redis-bundle storage-map remove redis-log - - name: Add a bind mount for logging in the redis bundle - command: pcs resource bundle update redis-bundle storage-map add id=redis-log source-dir=/var/log/containers/redis target-dir=/var/log/redis options=rw - - name: Update the redis bundle to use the new container image name - command: "pcs resource bundle update redis-bundle container image={{redis_docker_image_latest}}" - name: Retag the pacemaker image if containerized when: - step|int == 3 block: *redis_fetch_retag_container_tasks + fast_forward_upgrade_tasks: - when: - step|int == 0 diff --git a/deployment/haproxy/haproxy-pacemaker-puppet.yaml b/deployment/haproxy/haproxy-pacemaker-puppet.yaml index 9c5e9e0216..c975bd1aa4 100644 --- a/deployment/haproxy/haproxy-pacemaker-puppet.yaml +++ b/deployment/haproxy/haproxy-pacemaker-puppet.yaml @@ -405,36 +405,123 @@ outputs: - name: Haproxy fetch and retag container image for pacemaker when: step|int == 2 block: &haproxy_fetch_retag_container_tasks - - name: Get docker Haproxy image + - name: Get container haproxy image set_fact: - docker_image: {get_param: DockerHAProxyImage} - docker_image_latest: *haproxy_image_pcmklatest - - name: Get previous Haproxy image id - shell: "{{container_cli}} images | awk '/haproxy.* pcmklatest/{print $3}' | uniq" - register: haproxy_image_id - - block: - - name: Get a list of container using Haproxy image - shell: "{{container_cli}} ps -a -q -f 'ancestor={{haproxy_image_id.stdout}}'" - register: haproxy_containers_to_destroy - # It will be recreated with the delpoy step. - - name: Remove any container using the same Haproxy image - shell: "{{container_cli}} rm -fv {{item}}" - with_items: "{{ haproxy_containers_to_destroy.stdout_lines }}" - - name: Remove previous Haproxy images - shell: "{{container_cli}} rmi -f {{haproxy_image_id.stdout}}" - when: - - haproxy_image_id.stdout != '' - - name: Pull latest Haproxy images - command: "{{container_cli}} pull {{docker_image}}" - - name: Retag pcmklatest to latest Haproxy image - import_role: + haproxy_image: {get_param: DockerHAProxyImage} + haproxy_image_latest: *haproxy_image_pcmklatest + - name: Pull latest haproxy images + command: "{{container_cli}} pull {{haproxy_image}}" + - name: Get previous haproxy image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{haproxy_image_latest}}" + register: old_haproxy_image_id + failed_when: false + - name: Get new haproxy image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{haproxy_image}}" + register: new_haproxy_image_id + - name: Retag pcmklatest to latest haproxy image + include_role: name: tripleo-container-tag vars: - container_image: "{{ docker_image }}" - container_image_latest: "{{ docker_image_latest }}" - # Got to check that pacemaker_is_active is working fine with bundle. - # TODO: pacemaker_is_active resource doesn't support bundle. + container_image: "{{haproxy_image}}" + container_image_latest: "{{haproxy_image_latest}}" + when: + - old_haproxy_image_id.stdout != new_haproxy_image_id.stdout + - block: + - name: Get a list of container using haproxy image + shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_haproxy_image_id.stdout}}'" + register: haproxy_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same haproxy image + shell: "{{container_cli}} rm -fv {{item}}" + with_items: "{{ haproxy_containers_to_destroy.stdout_lines }}" + - name: Remove previous haproxy images + shell: "{{container_cli}} rmi -f {{old_haproxy_image_id.stdout}}" + when: + - old_haproxy_image_id.stdout != '' + - old_haproxy_image_id.stdout != new_haproxy_image_id.stdout + upgrade_tasks: + - name: Prepare switch of haproxy image name + when: + - step|int == 0 + block: + - name: Get haproxy image id currently used by pacemaker + shell: "pcs resource config haproxy-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'" + register: haproxy_image_current_res + failed_when: false + - name: Image facts for haproxy + set_fact: + haproxy_image_latest: *haproxy_image_pcmklatest + haproxy_image_current: "{{haproxy_image_current_res.stdout}}" + - name: Prepare the switch to new haproxy container image name in pacemaker + block: + - name: Temporarily tag the current haproxy image id with the upgraded image name + import_role: + name: tripleo-container-tag + vars: + container_image: "{{haproxy_image_current}}" + container_image_latest: "{{haproxy_image_latest}}" + pull_image: false + when: + - haproxy_image_current != '' + - haproxy_image_current != haproxy_image_latest + - name: Check haproxy cluster resource status + shell: pcs resource config haproxy-bundle + failed_when: false + register: haproxy_pcs_res_result + - name: Set upgrade haproxy facts + set_fact: + haproxy_pcs_res: "{{haproxy_pcs_res_result|succeeded}}" + is_haproxy_bootstrap_node: "{{haproxy_short_bootstrap_node_name|lower == ansible_hostname|lower}}" + + - name: Update haproxy pcs resource bundle for new container image + when: + - step|int == 1 + - is_haproxy_bootstrap_node|bool + - haproxy_pcs_res|bool + - haproxy_image_current != haproxy_image_latest + block: + - name: Disable the haproxy cluster resource before container upgrade + pacemaker_resource: + resource: haproxy-bundle + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + - name: Expose HAProxy stats socket on the host and mount TLS cert if needed + block: + - name: Check haproxy stats socket configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-var-lib']" + ignore_errors: true + register: haproxy_stats_exposed + - name: Check haproxy public certificate configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-cert']" + ignore_errors: true + register: haproxy_cert_mounted + - name: Add a bind mount for stats socket in the haproxy bundle + command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-var-lib source-dir=/var/lib/haproxy target-dir=/var/lib/haproxy options=rw + # rc == 6 means the configuration doesn't exist in the CIB + when: haproxy_stats_exposed.rc == 6 + - name: Set HAProxy public cert volume mount fact + set_fact: + haproxy_public_cert_path: {get_param: DeployedSSLCertificatePath} + haproxy_public_tls_enabled: {if: [public_tls_enabled, true, false]} + - name: Add a bind mount for public certificate in the haproxy bundle + command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-cert source-dir={{ haproxy_public_cert_path }} target-dir=/var/lib/kolla/config_files/src-tls/{{ haproxy_public_cert_path }} options=ro + when: + - haproxy_cert_mounted.rc == 6 + - haproxy_public_tls_enabled|bool + - name: Update the haproxy bundle to use the new container image name + command: "pcs resource bundle update haproxy-bundle container image={{haproxy_image_latest}}" + - name: Enable the haproxy cluster resource + pacemaker_resource: + resource: haproxy-bundle + state: enable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 - name: Create hiera data to upgrade haproxy in a stepwise manner. when: - step|int == 1 @@ -478,86 +565,6 @@ outputs: vars: tripleo_upgrade_key: haproxy_short_node_names_override when: haproxy_short_node_names_upgraded | length == haproxy_short_node_names | length - - name: upgrade haproxy bundle-init image. - when: - - step|int == 0 - tags: common - block: - - name: Get docker haproxy image - set_fact: - haproxy_docker_image_latest: *haproxy_image_pcmklatest - - name: Set HAProxy upgrade facts - block: *haproxy_update_upgrade_facts - - name: Prepare the switch to new haproxy container image name in pacemaker - block: - - name: Get haproxy image id currently used by pacemaker - shell: "{{container_cli}} images | awk '/haproxy.* pcmklatest/{print $3}' | uniq" - register: haproxy_current_pcmklatest_id - - name: Temporarily tag the current haproxy image id with the upgraded image name - import_role: - name: tripleo-container-tag - vars: - container_image: "{{haproxy_current_pcmklatest_id.stdout}}" - container_image_latest: "{{haproxy_docker_image_latest}}" - pull_image: false - when: haproxy_current_pcmklatest_id.stdout != '' - - name: Check haproxy-bundle cluster resource status - pacemaker_resource: - resource: haproxy-bundle - state: show - check_mode: false - ignore_errors: true - register: haproxy_pcs_res_result - - name: Set fact haproxy_pcs_res - set_fact: - haproxy_pcs_res: "{{haproxy_pcs_res_result|succeeded}}" - - name: Update haproxy pcs resource bundle for new container image - when: - - step|int == 1 - - is_haproxy_bootstrap_node - - haproxy_pcs_res|bool - block: - - name: Disable the haproxy cluster resource before container upgrade - pacemaker_resource: - resource: haproxy-bundle - state: disable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - - name: Expose HAProxy stats socket on the host and mount TLS cert if needed - block: - - name: Check haproxy stats socket configuration in pacemaker - command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-var-lib']" - ignore_errors: true - register: haproxy_stats_exposed - - name: Check haproxy public certificate configuration in pacemaker - command: cibadmin --query --xpath "//storage-mapping[@id='haproxy-cert']" - ignore_errors: true - register: haproxy_cert_mounted - - name: Add a bind mount for stats socket in the haproxy bundle - command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-var-lib source-dir=/var/lib/haproxy target-dir=/var/lib/haproxy options=rw - # rc == 6 means the configuration doesn't exist in the CIB - when: haproxy_stats_exposed.rc == 6 - - name: Set HAProxy public cert volume mount fact - set_fact: - haproxy_public_cert_path: {get_param: DeployedSSLCertificatePath} - haproxy_public_tls_enabled: {if: [public_tls_enabled, true, false]} - - name: Add a bind mount for public certificate in the haproxy bundle - command: pcs resource bundle update haproxy-bundle storage-map add id=haproxy-cert source-dir={{ haproxy_public_cert_path }} target-dir=/var/lib/kolla/config_files/src-tls/{{ haproxy_public_cert_path }} options=ro - when: - - haproxy_cert_mounted.rc == 6 - - haproxy_public_tls_enabled|bool - - name: Update the haproxy bundle to use the new container image name - command: "pcs resource bundle update haproxy-bundle container image={{haproxy_docker_image_latest}}" - - name: Enable the haproxy cluster resource - pacemaker_resource: - resource: haproxy-bundle - state: enable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - name: Retag the pacemaker image if containerized when: - step|int == 3 diff --git a/deployment/rabbitmq/rabbitmq-messaging-notify-pacemaker-puppet.yaml b/deployment/rabbitmq/rabbitmq-messaging-notify-pacemaker-puppet.yaml index e1ca8d921b..66392b6183 100644 --- a/deployment/rabbitmq/rabbitmq-messaging-notify-pacemaker-puppet.yaml +++ b/deployment/rabbitmq/rabbitmq-messaging-notify-pacemaker-puppet.yaml @@ -279,41 +279,115 @@ outputs: vars: container_image: {get_param: DockerRabbitmqImage} container_image_latest: *rabbitmq_image_pcmklatest + update_tasks: - name: Rabbit fetch and retag container image for pacemaker when: step|int == 2 block: &rabbitmq_fetch_retag_container_tasks - - name: Get docker Rabbitmq image + - name: Get container rabbitmq image set_fact: - docker_image: {get_param: DockerRabbitmqImage} - docker_image_latest: *rabbitmq_image_pcmklatest - - name: Get previous Rabbitmq image id - shell: "{{container_cli}} images | awk '/rabbitmq.* pcmklatest/{print $3}'" - register: rabbitmq_image_id - - block: - - name: Get a list of container using Rabbitmq image - shell: "{{container_cli}} ps -a -q -f 'ancestor={{rabbitmq_image_id.stdout}}'" - register: rabbitmq_containers_to_destroy - # It will be recreated with the delpoy step. - - name: Remove any container using the same Rabbitmq image - shell: "{{container_cli}} rm -fv {{item}}" - with_items: "{{ rabbitmq_containers_to_destroy.stdout_lines }}" - - name: Remove previous Rabbitmq images - shell: "{{container_cli}} rmi -f {{rabbitmq_image_id.stdout}}" - when: - - rabbitmq_image_id.stdout != '' - - name: Pull latest Rabbitmq images - command: "{{container_cli}} pull {{docker_image}}" - - name: Retag pcmklatest to latest Rabbitmq image - import_role: + rabbitmq_image: {get_param: DockerRabbitmqImage} + rabbitmq_image_latest: *rabbitmq_image_pcmklatest + - name: Pull latest rabbitmq images + command: "{{container_cli}} pull {{rabbitmq_image}}" + - name: Get previous rabbitmq image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{rabbitmq_image_latest}}" + register: old_rabbitmq_image_id + failed_when: false + - name: Get new rabbitmq image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{rabbitmq_image}}" + register: new_rabbitmq_image_id + - name: Retag pcmklatest to latest rabbitmq image + include_role: name: tripleo-container-tag vars: - container_image: "{{docker_image}}" - container_image_latest: "{{docker_image_latest}}" - pull_image: false - # Got to check that pacemaker_is_active is working fine with bundle. - # TODO: pacemaker_is_active resource doesn't support bundle. + container_image: "{{rabbitmq_image}}" + container_image_latest: "{{rabbitmq_image_latest}}" + when: + - old_rabbitmq_image_id.stdout != new_rabbitmq_image_id.stdout + - block: + - name: Get a list of container using rabbitmq image + shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_rabbitmq_image_id.stdout}}'" + register: rabbitmq_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same rabbitmq image + shell: "{{container_cli}} rm -fv {{item}}" + with_items: "{{ rabbitmq_containers_to_destroy.stdout_lines }}" + - name: Remove previous rabbitmq images + shell: "{{container_cli}} rmi -f {{old_rabbitmq_image_id.stdout}}" + when: + - old_rabbitmq_image_id.stdout != '' + - old_rabbitmq_image_id.stdout != new_rabbitmq_image_id.stdout + upgrade_tasks: + - name: Prepare switch of rabbitmq image name + when: + - step|int == 0 + block: + - name: Get rabbitmq image id currently used by pacemaker + shell: "pcs resource config rabbitmq-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'" + register: rabbitmq_image_current_res + failed_when: false + - name: Image facts for rabbitmq + set_fact: + rabbitmq_image_latest: *rabbitmq_image_pcmklatest + rabbitmq_image_current: "{{rabbitmq_image_current_res.stdout}}" + - name: Prepare the switch to new rabbitmq container image name in pacemaker + block: + - name: Temporarily tag the current rabbitmq image id with the upgraded image name + import_role: + name: tripleo-container-tag + vars: + container_image: "{{rabbitmq_image_current}}" + container_image_latest: "{{rabbitmq_image_latest}}" + pull_image: false + when: + - rabbitmq_image_current != '' + - rabbitmq_image_current != rabbitmq_image_latest + - name: Check rabbitmq cluster resource status + shell: pcs resource config rabbitmq-bundle + failed_when: false + register: rabbitmq_pcs_res_result + - name: Set fact rabbitmq_pcs_res + set_fact: + rabbitmq_pcs_res: "{{rabbitmq_pcs_res_result|succeeded}}" + - name: set is_notify_rabbitmq_bootstrap_node fact + set_fact: is_notify_rabbitmq_bootstrap_node={{oslo_messaging_notify_short_bootstrap_node_name|lower == ansible_hostname|lower}} + - name: Update rabbitmq-bundle pcs resource bundle for new container image + when: + - step|int == 1 + - is_notify_rabbitmq_bootstrap_node|bool + - rabbitmq_pcs_res|bool + - rabbitmq_image_current != rabbitmq_image_latest + block: + - name: Disable the rabbitmq cluster resource before container upgrade + pacemaker_resource: + resource: rabbitmq-bundle + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + - name: Move rabbitmq logging to /var/log/containers + block: + - name: Check rabbitmq logging configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='rabbitmq-log']" + ignore_errors: true + register: rabbitmq_logs_moved + - name: Add a bind mount for logging in the rabbitmq bundle + # rc == 6 means the configuration doesn't exist in the CIB + when: rabbitmq_logs_moved.rc == 6 + command: pcs resource bundle update rabbitmq-bundle storage-map add id=rabbitmq-log source-dir=/var/log/containers/rabbitmq target-dir=/var/log/rabbitmq options=rw + - name: Update the rabbitmq bundle to use the new container image name + command: "pcs resource bundle update rabbitmq-bundle container image={{rabbitmq_image_latest}}" + - name: Enable the rabbitmq cluster resource + pacemaker_resource: + resource: rabbitmq-bundle + state: enable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 - name: Create hiera data to upgrade oslo messaging notify in a stepwise manner. when: - step|int == 1 @@ -371,45 +445,6 @@ outputs: - oslo_messaging_notify_short_node_names_override - oslo_messaging_notify_node_names_override when: oslo_messaging_notify_short_node_names_upgraded | length == oslo_messaging_notify_node_names | length - - when: step|int == 0 - tags: common - block: - - name: Get docker Rabbitmq image - set_fact: - docker_image_latest: *rabbitmq_image_pcmklatest - - name: set is_notify_rabbitmq_bootstrap_node fact - set_fact: is_notify_rabbitmq_bootstrap_node={{oslo_messaging_notify_short_bootstrap_node_name|lower == ansible_hostname|lower}} - - name: Move rabbitmq logging to /var/log/containers - when: - - step|int == 1 - - is_notify_rabbitmq_bootstrap_node - block: - - name: Check rabbitmq logging configuration in pacemaker - command: cibadmin --query --xpath "//storage-mapping[@id='rabbitmq-log']" - ignore_errors: true - register: rabbitmq_logs_moved - - name: Change rabbitmq logging configuration in pacemaker - # rc == 6 means the configuration doesn't exist in the CIB - when: rabbitmq_logs_moved.rc == 6 - block: - - name: Disable the rabbitmq cluster resource - pacemaker_resource: - resource: rabbitmq - state: disable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - - name: Add a bind mount for logging in the rabbitmq bundle - command: pcs resource bundle update rabbitmq-bundle storage-map add id=rabbitmq-log source-dir=/var/log/containers/rabbitmq target-dir=/var/log/rabbitmq options=rw - - name: Enable the rabbitmq cluster resource - pacemaker_resource: - resource: rabbitmq - state: enable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - name: Retag the pacemaker image if containerized when: - step|int == 3 diff --git a/deployment/rabbitmq/rabbitmq-messaging-pacemaker-puppet.yaml b/deployment/rabbitmq/rabbitmq-messaging-pacemaker-puppet.yaml index 40d692f804..6a75b74ebe 100644 --- a/deployment/rabbitmq/rabbitmq-messaging-pacemaker-puppet.yaml +++ b/deployment/rabbitmq/rabbitmq-messaging-pacemaker-puppet.yaml @@ -279,40 +279,116 @@ outputs: vars: container_image: {get_param: DockerRabbitmqImage} container_image_latest: *rabbitmq_image_pcmklatest + update_tasks: - name: Rabbit fetch and retag container image for pacemaker when: step|int == 2 block: &rabbitmq_fetch_retag_container_tasks - - name: Get docker Rabbitmq image + - name: Get container rabbitmq image set_fact: - docker_image: {get_param: DockerRabbitmqImage} - docker_image_latest: *rabbitmq_image_pcmklatest - - name: Get previous Rabbitmq image id - shell: "{{container_cli}} images | awk '/rabbitmq.* pcmklatest/{print $3}' | uniq" - register: rabbitmq_image_id - - block: - - name: Get a list of container using Rabbitmq image - shell: "{{container_cli}} ps -a -q -f 'ancestor={{rabbitmq_image_id.stdout}}'" - register: rabbitmq_containers_to_destroy - # It will be recreated with the delpoy step. - - name: Remove any container using the same Rabbitmq image - shell: "{{container_cli}} rm -fv {{item}}" - with_items: "{{ rabbitmq_containers_to_destroy.stdout_lines }}" - - name: Remove previous Rabbitmq images - shell: "{{container_cli}} rmi -f {{rabbitmq_image_id.stdout}}" - when: - - rabbitmq_image_id.stdout != '' - - name: Pull latest Rabbitmq images - command: "{{container_cli}} pull {{docker_image}}" - - name: Retag pcmklatest to latest Rabbitmq image - import_role: + rabbitmq_image: {get_param: DockerRabbitmqImage} + rabbitmq_image_latest: *rabbitmq_image_pcmklatest + - name: Pull latest rabbitmq images + command: "{{container_cli}} pull {{rabbitmq_image}}" + - name: Get previous rabbitmq image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{rabbitmq_image_latest}}" + register: old_rabbitmq_image_id + failed_when: false + - name: Get new rabbitmq image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{rabbitmq_image}}" + register: new_rabbitmq_image_id + - name: Retag pcmklatest to latest rabbitmq image + include_role: name: tripleo-container-tag vars: - container_image: "{{docker_image}}" - container_image_latest: "{{docker_image_latest}}" - # Got to check that pacemaker_is_active is working fine with bundle. - # TODO: pacemaker_is_active resource doesn't support bundle. + container_image: "{{rabbitmq_image}}" + container_image_latest: "{{rabbitmq_image_latest}}" + when: + - old_rabbitmq_image_id.stdout != new_rabbitmq_image_id.stdout + - block: + - name: Get a list of container using rabbitmq image + shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_rabbitmq_image_id.stdout}}'" + register: rabbitmq_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same rabbitmq image + shell: "{{container_cli}} rm -fv {{item}}" + with_items: "{{ rabbitmq_containers_to_destroy.stdout_lines }}" + - name: Remove previous rabbitmq images + shell: "{{container_cli}} rmi -f {{old_rabbitmq_image_id.stdout}}" + when: + - old_rabbitmq_image_id.stdout != '' + - old_rabbitmq_image_id.stdout != new_rabbitmq_image_id.stdout + upgrade_tasks: + - name: Prepare switch of rabbitmq image name + when: + - step|int == 0 + block: + - name: Get rabbitmq image id currently used by pacemaker + shell: "pcs resource config rabbitmq-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'" + register: rabbitmq_image_current_res + failed_when: false + - name: Image facts for rabbitmq + set_fact: + rabbitmq_image_latest: *rabbitmq_image_pcmklatest + rabbitmq_image_current: "{{rabbitmq_image_current_res.stdout}}" + - name: Prepare the switch to new rabbitmq container image name in pacemaker + block: + - name: Temporarily tag the current rabbitmq image id with the upgraded image name + import_role: + name: tripleo-container-tag + vars: + container_image: "{{rabbitmq_image_current}}" + container_image_latest: "{{rabbitmq_image_latest}}" + pull_image: false + when: + - rabbitmq_image_current != '' + - rabbitmq_image_current != rabbitmq_image_latest + - name: Check rabbitmq cluster resource status + shell: pcs resource config rabbitmq-bundle + failed_when: false + register: rabbitmq_pcs_res_result + - name: Set fact rabbitmq_pcs_res + set_fact: + rabbitmq_pcs_res: "{{rabbitmq_pcs_res_result|succeeded}}" + - name: set is_rabbitmq_bootstrap_node fact + tags: common + set_fact: is_rabbitmq_bootstrap_node={{rabbitmq_short_bootstrap_node_name|lower == ansible_hostname|lower}} + - name: Update rabbitmq-bundle pcs resource bundle for new container image + when: + - step|int == 1 + - is_rabbitmq_bootstrap_node|bool + - rabbitmq_pcs_res|bool + - rabbitmq_image_current != rabbitmq_image_latest + block: + - name: Disable the rabbitmq cluster resource before container upgrade + pacemaker_resource: + resource: rabbitmq-bundle + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + - name: Move rabbitmq logging to /var/log/containers + block: + - name: Check rabbitmq logging configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='rabbitmq-log']" + ignore_errors: true + register: rabbitmq_logs_moved + - name: Add a bind mount for logging in the rabbitmq bundle + # rc == 6 means the configuration doesn't exist in the CIB + when: rabbitmq_logs_moved.rc == 6 + command: pcs resource bundle update rabbitmq-bundle storage-map add id=rabbitmq-log source-dir=/var/log/containers/rabbitmq target-dir=/var/log/rabbitmq options=rw + - name: Update the rabbitmq bundle to use the new container image name + command: "pcs resource bundle update rabbitmq-bundle container image={{rabbitmq_image_latest}}" + - name: Enable the rabbitmq cluster resource + pacemaker_resource: + resource: rabbitmq-bundle + state: enable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 - name: Create hiera data to upgrade rabbitmq in a stepwise manner. when: - step|int == 1 @@ -356,71 +432,11 @@ outputs: vars: tripleo_upgrade_key: rabbitmq_short_node_names_override when: rabbitmq_short_node_names_upgraded | length == rabbitmq_short_node_names | length - - name: Get docker Rabbitmq image - set_fact: - rabbitmq_docker_image_latest: *rabbitmq_image_pcmklatest - - name: set is_rabbitmq_bootstrap_node fact - set_fact: is_rabbitmq_bootstrap_node={{rabbitmq_short_bootstrap_node_name|lower == ansible_hostname|lower}} - - name: Prepare the switch to new rabbitmq container image name in pacemaker - when: - - step|int == 0 - block: - - name: Get rabbitmq image id currently used by pacemaker - shell: "{{container_cli}} images | awk '/rabbitmq.* pcmklatest/{print $3}' | uniq" - register: rabbitmq_current_pcmklatest_id - - name: Temporarily tag the current rabbitmq image id with the upgraded image name - import_role: - name: tripleo-container-tag - vars: - container_image: "{{rabbitmq_current_pcmklatest_id.stdout}}" - container_image_latest: "{{rabbitmq_docker_image_latest}}" - pull_image: false - when: rabbitmq_current_pcmklatest_id.stdout != '' - - name: Check rabbitmq-bundle cluster resource status - pacemaker_resource: - resource: rabbitmq-bundle - state: show - check_mode: false - ignore_errors: true - register: rabbit_pcs_res - - name: Update rabbitmq-bundle pcs resource bundle for new container image - when: - - step|int == 1 - - is_rabbitmq_bootstrap_node - - rabbit_pcs_res|succeeded - block: - - name: Disable the rabbitmq cluster resource before container upgrade - pacemaker_resource: - resource: rabbitmq-bundle - state: disable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - - name: Move rabbitmq logging to /var/log/containers - block: - - name: Check rabbitmq logging configuration in pacemaker - command: cibadmin --query --xpath "//storage-mapping[@id='rabbitmq-log']" - ignore_errors: true - register: rabbitmq_logs_moved - - name: Add a bind mount for logging in the rabbitmq bundle - # rc == 6 means the configuration doesn't exist in the CIB - when: rabbitmq_logs_moved.rc == 6 - command: pcs resource bundle update rabbitmq-bundle storage-map add id=rabbitmq-log source-dir=/var/log/containers/rabbitmq target-dir=/var/log/rabbitmq options=rw - - name: Update the rabbitmq bundle to use the new container image name - command: "pcs resource bundle update rabbitmq-bundle container image={{rabbitmq_docker_image_latest}}" - - name: Enable the rabbitmq cluster resource - pacemaker_resource: - resource: rabbitmq-bundle - state: enable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - name: Retag the pacemaker image if containerized when: - step|int == 3 block: *rabbitmq_fetch_retag_container_tasks + fast_forward_upgrade_tasks: - when: - step|int == 0 diff --git a/deployment/rabbitmq/rabbitmq-messaging-rpc-pacemaker-puppet.yaml b/deployment/rabbitmq/rabbitmq-messaging-rpc-pacemaker-puppet.yaml index 818786b749..6b43397fd2 100644 --- a/deployment/rabbitmq/rabbitmq-messaging-rpc-pacemaker-puppet.yaml +++ b/deployment/rabbitmq/rabbitmq-messaging-rpc-pacemaker-puppet.yaml @@ -279,41 +279,116 @@ outputs: vars: container_image: {get_param: DockerRabbitmqImage} container_image_latest: *rabbitmq_image_pcmklatest + update_tasks: - name: Rabbit fetch and retag container image for pacemaker when: step|int == 2 block: &rabbitmq_fetch_retag_container_tasks - - name: Get docker Rabbitmq image + - name: Get container rabbitmq image set_fact: - docker_image: {get_param: DockerRabbitmqImage} - docker_image_latest: *rabbitmq_image_pcmklatest - - name: Get previous Rabbitmq image id - shell: "{{container_cli}} images | awk '/rabbitmq.* pcmklatest/{print $3}' | uniq" - register: rabbitmq_image_id - - block: - - name: Get a list of container using Rabbitmq image - shell: "{{container_cli}} ps -a -q -f 'ancestor={{rabbitmq_image_id.stdout}}'" - register: rabbitmq_containers_to_destroy - # It will be recreated with the delpoy step. - - name: Remove any container using the same Rabbitmq image - shell: "{{container_cli}} rm -fv {{item}}" - with_items: "{{ rabbitmq_containers_to_destroy.stdout_lines }}" - - name: Remove previous Rabbitmq images - shell: "{{container_cli}} rmi -f {{rabbitmq_image_id.stdout}}" - when: - - rabbitmq_image_id.stdout != '' - - name: Pull latest Rabbitmq images - command: "{{container_cli}} pull {{docker_image}}" - - name: Retag pcmklatest to latest Rabbitmq image - import_role: + rabbitmq_image: {get_param: DockerRabbitmqImage} + rabbitmq_image_latest: *rabbitmq_image_pcmklatest + - name: Pull latest rabbitmq images + command: "{{container_cli}} pull {{rabbitmq_image}}" + - name: Get previous rabbitmq image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{rabbitmq_image_latest}}" + register: old_rabbitmq_image_id + failed_when: false + - name: Get new rabbitmq image id + shell: "{{container_cli}} inspect --format '{{'{{'}}.Id{{'}}'}}' {{rabbitmq_image}}" + register: new_rabbitmq_image_id + - name: Retag pcmklatest to latest rabbitmq image + include_role: name: tripleo-container-tag vars: - container_image: "{{docker_image}}" - container_image_latest: "{{docker_image_latest}}" - # Got to check that pacemaker_is_active is working fine with bundle. - # TODO: pacemaker_is_active resource doesn't support bundle. + container_image: "{{rabbitmq_image}}" + container_image_latest: "{{rabbitmq_image_latest}}" + when: + - old_rabbitmq_image_id.stdout != new_rabbitmq_image_id.stdout + - block: + - name: Get a list of container using rabbitmq image + shell: "{{container_cli}} ps -a -q -f 'ancestor={{old_rabbitmq_image_id.stdout}}'" + register: rabbitmq_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same rabbitmq image + shell: "{{container_cli}} rm -fv {{item}}" + with_items: "{{ rabbitmq_containers_to_destroy.stdout_lines }}" + - name: Remove previous rabbitmq images + shell: "{{container_cli}} rmi -f {{old_rabbitmq_image_id.stdout}}" + when: + - old_rabbitmq_image_id.stdout != '' + - old_rabbitmq_image_id.stdout != new_rabbitmq_image_id.stdout + upgrade_tasks: - - name: Create hiera data to upgrade oslo_messaging_rpc in a stepwise manner. + - name: Prepare switch of rabbitmq image name + when: + - step|int == 0 + block: + - name: Get rabbitmq image id currently used by pacemaker + shell: "pcs resource config rabbitmq-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'" + register: rabbitmq_image_current_res + failed_when: false + - name: Image facts for rabbitmq + set_fact: + rabbitmq_image_latest: *rabbitmq_image_pcmklatest + rabbitmq_image_current: "{{rabbitmq_image_current_res.stdout}}" + - name: Prepare the switch to new rabbitmq container image name in pacemaker + block: + - name: Temporarily tag the current rabbitmq image id with the upgraded image name + import_role: + name: tripleo-container-tag + vars: + container_image: "{{rabbitmq_image_current}}" + container_image_latest: "{{rabbitmq_image_latest}}" + pull_image: false + when: + - rabbitmq_image_current != '' + - rabbitmq_image_current != rabbitmq_image_latest + - name: Check rabbitmq cluster resource status + shell: pcs resource config rabbitmq-bundle + failed_when: false + register: rabbitmq_pcs_res_result + - name: Set fact rabbitmq_pcs_res + set_fact: + rabbitmq_pcs_res: "{{rabbitmq_pcs_res_result|succeeded}}" + - name: set is_rpc_rabbitmq_bootstrap_node fact + set_fact: is_rpc_rabbitmq_bootstrap_node={{oslo_messaging_rpc_short_bootstrap_node_name|lower == ansible_hostname|lower}} + - name: Update rabbitmq-bundle pcs resource bundle for new container image + when: + - step|int == 1 + - is_rpc_rabbitmq_bootstrap_node|bool + - rabbitmq_pcs_res|bool + - rabbitmq_image_current != rabbitmq_image_latest + block: + - name: Disable the rabbitmq cluster resource before container upgrade + pacemaker_resource: + resource: rabbitmq-bundle + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + - name: Move rabbitmq logging to /var/log/containers + block: + - name: Check rabbitmq logging configuration in pacemaker + command: cibadmin --query --xpath "//storage-mapping[@id='rabbitmq-log']" + ignore_errors: true + register: rabbitmq_logs_moved + - name: Add a bind mount for logging in the rabbitmq bundle + # rc == 6 means the configuration doesn't exist in the CIB + when: rabbitmq_logs_moved.rc == 6 + command: pcs resource bundle update rabbitmq-bundle storage-map add id=rabbitmq-log source-dir=/var/log/containers/rabbitmq target-dir=/var/log/rabbitmq options=rw + - name: Update the rabbitmq bundle to use the new container image name + command: "pcs resource bundle update rabbitmq-bundle container image={{rabbitmq_image_latest}}" + - name: Enable the rabbitmq cluster resource + pacemaker_resource: + resource: rabbitmq-bundle + state: enable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + - name: Create hiera data to upgrade oslo messaging rpc in a stepwise manner. when: - step|int == 1 block: @@ -337,8 +412,8 @@ outputs: msg: "Prepare oslo_messaging_rpc upgrade for {{ oslo_messaging_rpc_short_node_names_upgraded }}" - fail: msg: > - You can't upgrade oslo_messaging_rpc without staged - upgrade. You need to use the limit option in order + You can't upgrade oslo_messaging_rpc without + staged upgrade. You need to use the limit option in order to do so. when: >- oslo_messaging_rpc_short_node_names_upgraded is not defined or @@ -370,71 +445,11 @@ outputs: - oslo_messaging_rpc_short_node_names_override - oslo_messaging_rpc_node_names_override when: oslo_messaging_rpc_short_node_names_upgraded | length == oslo_messaging_rpc_node_names | length - - name: Get docker Rabbitmq image - set_fact: - rabbitmq_docker_image_latest: *rabbitmq_image_pcmklatest - - name: set is_rpc_rabbitmq_bootstrap_node fact - set_fact: is_rpc_rabbitmq_bootstrap_node={{oslo_messaging_rpc_short_bootstrap_node_name|lower == ansible_hostname|lower}} - - name: Prepare the switch to new rabbitmq container image name in pacemaker - when: - - step|int == 0 - block: - - name: Get rabbitmq image id currently used by pacemaker - shell: "{{container_cli}} images | awk '/rabbitmq.* pcmklatest/{print $3}' | uniq" - register: rabbitmq_current_pcmklatest_id - - name: Temporarily tag the current rabbitmq image id with the upgraded image name - import_role: - name: tripleo-container-tag - vars: - container_image: "{{rabbitmq_current_pcmklatest_id.stdout}}" - container_image_latest: "{{rabbitmq_docker_image_latest}}" - pull_image: false - when: rabbitmq_current_pcmklatest_id.stdout != '' - - name: Check rabbitmq-bundle cluster resource status - pacemaker_resource: - resource: rabbitmq-bundle - state: show - check_mode: false - ignore_errors: true - register: rabbit_pcs_res - - name: Update rabbitmq-bundle pcs resource bundle for new container image - when: - - step|int == 1 - - is_rpc_rabbitmq_bootstrap_node - - rabbit_pcs_res|succeeded - block: - - name: Disable the rabbitmq cluster resource before container upgrade - pacemaker_resource: - resource: rabbitmq-bundle - state: disable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - - name: Move rabbitmq logging to /var/log/containers - block: - - name: Check rabbitmq logging configuration in pacemaker - command: cibadmin --query --xpath "//storage-mapping[@id='rabbitmq-log']" - ignore_errors: true - register: rabbitmq_logs_moved - - name: Add a bind mount for logging in the rabbitmq bundle - # rc == 6 means the configuration doesn't exist in the CIB - when: rabbitmq_logs_moved.rc == 6 - command: pcs resource bundle update rabbitmq-bundle storage-map add id=rabbitmq-log source-dir=/var/log/containers/rabbitmq target-dir=/var/log/rabbitmq options=rw - - name: Update the rabbitmq bundle to use the new container image name - command: "pcs resource bundle update rabbitmq-bundle container image={{rabbitmq_docker_image_latest}}" - - name: Enable the rabbitmq cluster resource - pacemaker_resource: - resource: rabbitmq-bundle - state: enable - wait_for_resource: true - register: output - retries: 5 - until: output.rc == 0 - name: Retag the pacemaker image if containerized when: - step|int == 3 block: *rabbitmq_fetch_retag_container_tasks + fast_forward_upgrade_tasks: - when: - step|int == 0