From c90e0a7197e5f84d4f0cf1e284866eba48c093d4 Mon Sep 17 00:00:00 2001 From: Numan Siddique Date: Fri, 8 Jun 2018 00:27:16 +0530 Subject: [PATCH] Add missing 'update_tasks' to the OVN dbs pacemaker service Presently the OVN dbs pacemaker service file doesn't include the update_tasks section and it neither tags the docker image to be used with pcmklatest. Because of which when a minor update is run, the latest docker image for ovn-dbs-bundle is never pulled. This patch addresses these issue. It does 3 things 1. It adds update_tasks section similar to other pacemaker bundle services. It adds additional few steps in step 5 (which is not the case with other pacemaker bundle services) to update the ovn bundle service to use the image tagged with pcmklatest if it not yet using. These additional steps are required for OVN deployments deployed before this patch and wants to do a minor stack update. 2. It adds the required tasks in "upgrade_tasks" section similar to other pacemaker bundle services. 3. Adds docker_config step to tag the image with pcmklatest tag. Change-Id: Idbda3bedff57376b74269ab3470d2324b804ffd4 Closes-bug: #1775686 --- docker/services/pacemaker/ovn-dbs.yaml | 169 +++++++++++++++++- ...x-minor-update-issue-3a1206549e3b75aa.yaml | 7 + 2 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/ovn-dbs-ha-fix-minor-update-issue-3a1206549e3b75aa.yaml diff --git a/docker/services/pacemaker/ovn-dbs.yaml b/docker/services/pacemaker/ovn-dbs.yaml index 4dfab0bcb8..3dce93261e 100644 --- a/docker/services/pacemaker/ovn-dbs.yaml +++ b/docker/services/pacemaker/ovn-dbs.yaml @@ -77,7 +77,13 @@ outputs: config_settings: map_merge: - get_attr: [OVNDbsBase, role_data, config_settings] - - tripleo::profile::pacemaker::ovn_dbs_bundle::ovn_dbs_docker_image: {get_param: DockerOvnDbsImage} + - tripleo::profile::pacemaker::ovn_dbs_bundle::ovn_dbs_docker_image: &ovn_dbs_image_pcmklatest + list_join: + - ':' + - - yaql: + data: {get_param: DockerOvnDbsImage} + expression: $.data.rightSplit(separator => ":", maxSplits => 1)[0] + - 'pcmklatest' - tripleo::profile::pacemaker::ovn_dbs_bundle::nb_db_port: {get_param: OVNNorthboundServerPort} - tripleo::profile::pacemaker::ovn_dbs_bundle::sb_db_port: {get_param: OVNSouthboundServerPort} logging_source: {get_attr: [OVNDbsBase, role_data, logging_source]} @@ -104,6 +110,29 @@ outputs: optional: true docker_config_scripts: {get_attr: [ContainersCommon, docker_config_scripts]} docker_config: + step_2: + ovn_dbs_image_tag: + start_order: 0 + detach: false + net: host + user: root + command: + - '/bin/bash' + - '-c' + - str_replace: + template: + "/usr/bin/docker tag 'OVN_DBS_IMAGE' 'OVN_DBS_IMAGE_PCMKLATEST'" + params: + OVN_DBS_IMAGE: {get_param: DockerOvnDbsImage} + OVN_DBS_IMAGE_PCMKLATEST: *ovn_dbs_image_pcmklatest + image: {get_param: DockerOvnDbsImage} + volumes: + - /etc/hosts:/etc/hosts:ro + - /etc/localtime:/etc/localtime:ro + - /dev/shm:/dev/shm:rw + - /etc/sysconfig/docker:/etc/sysconfig/docker:ro + - /usr/bin:/usr/bin:ro + - /var/run/docker.sock:/var/run/docker.sock:rw step_3: ovn_dbs_init_bundle: start_order: 1 @@ -142,7 +171,145 @@ outputs: Log files from openvswitch containers can be found under /var/log/containers/openvswitch. ignore_errors: true + update_tasks: + - name: Get docker ovn-dbs image + set_fact: + ovn_dbs_docker_image: {get_param: DockerOvnDbsImage} + ovn_dbs_docker_image_latest: *ovn_dbs_image_pcmklatest + - name: Check for ovn-dbs log file + stat: + path: /var/log/containers/openvswitch/ovsdb-server-nb.log + register: ovn_dbs_log_file + - name: Check if ovn-dbs is already containerized + set_fact: + ovn_dbs_containerized: "{{ovn_dbs_log_file.stat.exists | default(false)}}" + - name: get bootstrap nodeid + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: ovn-dbs fetch and retag container image for pacemaker + when: + - step|int == 3 + - ovn_dbs_containerized|bool + block: &ovn_dbs_fetch_retag_container_tasks + - name: Get previous ovn-dbs image id + shell: "docker images | awk '/ovn.* pcmklatest/{print $3}' | uniq" + register: ovn_dbs_image_id + - block: + - name: Get a list of container using ovn-dbs image + shell: "docker ps -a -q -f 'ancestor={{ovn_dbs_image_id.stdout}}'" + register: ovn_dbs_containers_to_destroy + # It will be recreated with the deploy step. + - name: Remove any container using the same ovn-dbs image + shell: "docker rm -fv {{item}}" + with_items: "{{ ovn_dbs_containers_to_destroy.stdout_lines }}" + - name: Remove previous ovn-dbs images + shell: "docker rmi -f {{ovn_dbs_image_id.stdout}}" + when: + - ovn_dbs_image_id.stdout != '' + - name: Pull latest ovn-dbs images + command: "docker pull {{ovn_dbs_docker_image}}" + - name: Retag pcmklatest to latest ovn-dbs image + shell: "docker tag {{ovn_dbs_docker_image}} {{ovn_dbs_docker_image_latest}}" + # Got to check that pacemaker_is_active is working fine with bundle. + # TODO: pacemaker_is_active resource doesn't support bundle. + # When ovn-dbs-bundle support was added, we didn't tag the ovn-dbs image + # with pcmklatest. So, when update is run for the first time we need to + # update the ovn-dbs-bundle resource to use the 'pcmklatest' tagged image. + # See https://bugzilla.redhat.com/show_bug.cgi?id=1586132. + # Step 3 (see above) takes care of tagging the image. + - name: Update ovn-dbs-bundle resource to use pcmklatest tag image if not used + when: + - step|int == 5 + - ovn_dbs_containerized|bool + - is_bootstrap_node + block: + - name: Get the present image used by ovn-dbs-bundle + shell: "pcs resource show ovn-dbs-bundle | grep image | awk '{ split($2, image, \"=\"); print image[2] }'" + register: ovn_dbs_current_image + - block: &ovn_dbs_update_bundle_with_new_image + - name: Disable the ovn-dbs-bundle resource before updating the image + pacemaker_resource: + resource: ovn-dbs-bundle + state: disable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + - name: Update the ovn-dbs-bundle to use the new container image name + command: "pcs resource bundle update ovn-dbs-bundle container image={{ovn_dbs_docker_image_latest}}" + - name: Enable the ovn-dbs-bundle resource + pacemaker_resource: + resource: ovn-dbs-bundle + state: enable + wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 + when: + - ovn_dbs_current_image.stdout != ovn_dbs_docker_image_latest upgrade_tasks: - name: Stop and disable ovn-northd service when: step|int == 1 service: name=ovn-northd state=stopped enabled=no + ignore_errors: true + - name: Check for ovn-dbs log file + stat: + path: /var/log/containers/openvswitch/ovsdb-server-nb.log + register: ovn_dbs_log_file + - name: Check if ovn-dbs is already containerized + set_fact: + ovn_dbs_containerized: "{{ovn_dbs_log_file.stat.exists | default(false)}}" + - name: Get docker ovn-dbs image + set_fact: + ovn_dbs_docker_image: {get_param: DockerOvnDbsImage} + ovn_dbs_docker_image_latest: *ovn_dbs_image_pcmklatest + - name: get bootstrap nodeid + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Prepare the switch to new ovn-dbs container image name in pacemaker + when: + - step|int == 0 + - ovn_dbs_containerized|bool + block: + - name: Get ovn-dbs image id currently used by pacemaker + shell: "docker images | awk '/ovn.* pcmklatest/{print $3}' | uniq" + register: ovn_dbs_current_pcmklatest_id + - name: Temporarily tag the current ovn-dbs pcmklatest image id with the upgraded image name + shell: "docker tag {{ovn_dbs_current_pcmklatest_id.stdout}} {{ovn_dbs_docker_image_latest}}" + when: ovn_dbs_current_pcmklatest_id.stdout != '' + # If ovn-dbs image is not tagged with pcmklatest, then create a new + # tag. This could happen if the stack is upgraded without updating the stack before. + # In the next step, the block 'ovn_dbs_update_bundle_with_new_image' + # will update the ovn-dbs-bundle resource to use the tagged image. + # And in step 3, we will fetch the latest image. + - block: + - name: Get the present image used by ovn-dbs-bundle + shell: "pcs resource show ovn-dbs-bundle | grep image | awk '{ split($2, image, \"=\"); print image[2] }'" + register: ovn_dbs_current_image + - name: Tag the current image with pcmklatest tag + shell: "docker tag {{ovn_dbs_current_image.stdout}} {{ovn_dbs_docker_image_latest}}" + when: + - ovn_dbs_current_pcmklatest_id.stdout == '' + - name: Check ovn-dbs-bundle cluster resource status + pacemaker_resource: + resource: ovn-dbs-bundle + state: show + check_mode: false + ignore_errors: true + register: ovndbs_pcs_res + - name: Update ovn-bundle pcs resource bundle for new container image + when: + - step|int == 1 + - ovn_dbs_containerized|bool + - is_bootstrap_node + - ovndbs_pcs_res|succeeded + block: *ovn_dbs_update_bundle_with_new_image + - name: Retag the pacemaker image if containerized + when: + - step|int == 3 + - ovn_dbs_containerized|bool + block: *ovn_dbs_fetch_retag_container_tasks diff --git a/releasenotes/notes/ovn-dbs-ha-fix-minor-update-issue-3a1206549e3b75aa.yaml b/releasenotes/notes/ovn-dbs-ha-fix-minor-update-issue-3a1206549e3b75aa.yaml new file mode 100644 index 0000000000..1f08c1fe89 --- /dev/null +++ b/releasenotes/notes/ovn-dbs-ha-fix-minor-update-issue-3a1206549e3b75aa.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes minor updates issue for ovn dbs pacemaker bundle resource by tagging + the docker image used for ovn dbs pacemaker resource with pcmklatest and + adding required missing tasks in "update_tasks" and "upgrade_tasks" + section of the service file.