From e81878bc041a8d579844b1e15f43b24cfb06cb25 Mon Sep 17 00:00:00 2001 From: Jiri Stransky Date: Wed, 19 Sep 2018 17:00:11 +0200 Subject: [PATCH] Run online migrations via "external-upgrade run" This will pull the online data migrations out of the upgrade maintenance window and let them be performed after the main upgrade phase while the cloud is already operational. The online part of the service upgrades can be run using: openstack overcloud external-upgrade run --tags online_upgrade or per-service like: openstack overcloud external-upgrade run --tags online_upgrade_nova openstack overcloud external-upgrade run --tags online_upgrade_cinder openstack overcloud external-upgrade run --tags online_upgrade_ironic Change-Id: I35c8d9985df21b3084fba558687e1f408e5a0878 Closes-Bug: #1793332 --- docker/services/cinder-api.yaml | 60 +++++++++---------- docker/services/ironic-api.yaml | 39 +++++------- docker/services/nova-api.yaml | 31 +++++----- ...ion-external-upgrade-5093de6bd8993b5c.yaml | 16 +++++ 4 files changed, 71 insertions(+), 75 deletions(-) create mode 100644 releasenotes/notes/online-migration-external-upgrade-5093de6bd8993b5c.yaml diff --git a/docker/services/cinder-api.yaml b/docker/services/cinder-api.yaml index f84ce7821b..1d2c39fdd3 100644 --- a/docker/services/cinder-api.yaml +++ b/docker/services/cinder-api.yaml @@ -179,26 +179,6 @@ outputs: - '' environment: - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS - step_5: - cinder_api_online_migrations: - start_order: 2 - image: *cinder_api_image - net: host - privileged: false - detach: false - user: root - volumes: - list_concat: - - {get_attr: [ContainersCommon, volumes]} - - - - /var/lib/config-data/cinder/etc/my.cnf.d/tripleo.cnf:/etc/my.cnf.d/tripleo.cnf:ro - - /var/lib/config-data/cinder/etc/cinder/:/etc/cinder/:ro - - /var/log/containers/cinder:/var/log/cinder - - /var/log/containers/httpd/cinder-api:/var/log/httpd - command: - - '/usr/bin/bootstrap_host_exec' - - 'cinder_api' - - "su cinder -s /bin/bash -c 'cinder-manage db online_data_migrations'" cinder_api_cron: image: *cinder_api_image net: host @@ -261,31 +241,35 @@ outputs: - name: Set fact cinder_api_enabled set_fact: cinder_api_enabled: "{{ cinder_api_enabled_result.rc == 0 }}" + - name: Check if httpd service is running + command: systemctl is-active --quiet httpd + ignore_errors: True + register: httpd_running_result + when: httpd_running is undefined + - name: Set fact httpd_running + set_fact: + httpd_running: "{{ httpd_running_result.rc == 0 }}" + when: httpd_running is undefined + - name: check for cinder_api running under apache (post upgrade) + shell: "httpd -t -D DUMP_VHOSTS | grep -q cinder" + register: cinder_api_apache + ignore_errors: true - name: "PreUpgrade step0,validation: Check service openstack-cinder-api is running" shell: systemctl is-active --quiet openstack-cinder-api when: cinder_api_enabled|bool tags: validation - # NOTE: In puppet service variant, we ran release N online - # migrations before upgrade to release N+1. In docker - # service variant, we run release N online migrations - # after upgrade to release N. This means that during - # switch from non-containerized to containerized, we need - # to run both of these. - - name: Online data migration for Cinder before switching to containers + - name: Ensure all online data migrations for Cinder have been applied + command: docker exec cinder_api cinder-manage db online_data_migrations tags: pre-upgrade when: - is_bootstrap_node|bool - - cinder_api_enabled|bool - command: cinder-manage db online_data_migrations + - not (cinder_api_apache|bool and httpd_running|bool) + - not cinder_api_enabled|bool - when: step|int == 2 block: - name: Stop and disable cinder_api service (pre-upgrade not under httpd) service: name=openstack-cinder-api state=stopped enabled=no when: cinder_api_enabled|bool - - name: check for cinder_api running under apache (post upgrade) - shell: "httpd -t -D DUMP_VHOSTS | grep -q cinder" - register: cinder_api_apache - ignore_errors: true - name: Stop and disable cinder_api service service: name=httpd state=stopped enabled=no when: cinder_api_apache.rc == 0 @@ -302,6 +286,16 @@ outputs: package: name=httpd state=removed ignore_errors: True when: remove_httpd_package|bool + external_upgrade_tasks: + - when: step|int == 1 + block: + - name: Online data migration for Cinder + command: docker exec cinder_api cinder-manage db online_data_migrations + delegate_to: "{{ groups['cinder_api'][0] }}" + become: true + tags: + - online_upgrade + - online_upgrade_cinder fast_forward_upgrade_tasks: - when: - step|int == 0 diff --git a/docker/services/ironic-api.yaml b/docker/services/ironic-api.yaml index 7cf09a9d44..f234cf6998 100644 --- a/docker/services/ironic-api.yaml +++ b/docker/services/ironic-api.yaml @@ -138,22 +138,6 @@ outputs: - /var/log/containers/httpd/ironic-api:/var/log/httpd environment: - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS - step_5: - ironic_online_migrations: - start_order: 2 - image: *ironic_api_image - net: host - privileged: false - detach: false - user: root - volumes: - list_concat: - - {get_attr: [ContainersCommon, volumes]} - - - - /var/lib/config-data/ironic_api/etc/ironic:/etc/ironic:ro - - /var/log/containers/ironic:/var/log/ironic - - /var/log/containers/httpd/ironic-api:/var/log/httpd - command: "/usr/bin/bootstrap_host_exec ironic_api su ironic -s /bin/bash -c 'ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations'" host_prep_tasks: - name: create persistent logs directory file: @@ -213,18 +197,13 @@ outputs: when: - ironic_httpd_enabled|bool - httpd_running|bool - # NOTE: In puppet service variant, we ran release N online - # migrations before upgrade to release N+1. In docker - # service variant, we run release N online migrations - # after upgrade to release N. This means that during - # switch from non-containerized to containerized, we need - # to run both of these. - - name: Online data migration for Ironic before switching to containers + - name: Ensure all online data migrations for Ironic have been applied + command: docker exec ironic_api ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations tags: pre-upgrade when: - is_bootstrap_node|bool - - (ironic_httpd_enabled|bool and httpd_running|bool) or ironic_api_enabled|bool - command: ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations + - not (ironic_httpd_enabled|bool and httpd_running|bool) + - not ironic_api_enabled|bool - when: step|int == 2 block: - name: Stop and disable ironic_api service @@ -245,6 +224,16 @@ outputs: package: name=openstack-ironic-api state=removed ignore_errors: True when: remove_ironic_api_package|bool + external_upgrade_tasks: + - when: step|int == 1 + block: + - name: Online data migration for Ironic + command: docker exec ironic_api ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations + delegate_to: "{{ groups['ironic_api'][0] }}" + become: true + tags: + - online_upgrade + - online_upgrade_ironic fast_forward_upgrade_tasks: - when: - step|int == 0 diff --git a/docker/services/nova-api.yaml b/docker/services/nova-api.yaml index 84abc41a09..63b010adf9 100644 --- a/docker/services/nova-api.yaml +++ b/docker/services/nova-api.yaml @@ -320,14 +320,6 @@ outputs: - '' - - 'TRIPLEO_DEPLOY_IDENTIFIER=' - {get_param: DeployIdentifier} - nova_online_migrations: - start_order: 2 - image: *nova_api_image - net: host - detach: false - volumes: *nova_api_bootstrap_volumes - user: root - command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage db online_data_migrations'" metadata_settings: get_attr: [NovaApiBase, role_data, metadata_settings] host_prep_tasks: {get_attr: [NovaApiLogging, host_prep_tasks]} @@ -371,18 +363,13 @@ outputs: when: - nova_api_httpd_enabled|bool - httpd_running|bool - # NOTE: In puppet service variant, we ran release N online - # migrations before upgrade to release N+1. In docker - # service variant, we run release N online migrations - # after upgrade to release N. This means that during - # switch from non-containerized to containerized, we need - # to run both of these. - - name: Online data migration for Nova before switching to containers + - name: Ensure all online data migrations for Nova have been applied + command: docker exec nova_api nova-manage db online_data_migrations tags: pre-upgrade when: - is_bootstrap_node|bool - - (nova_api_httpd_enabled|bool and httpd_running|bool) or nova_api_enabled|bool - command: nova-manage db online_data_migrations + - not (nova_api_httpd_enabled|bool and httpd_running|bool) + - not nova_api_enabled|bool - when: step|int == 2 block: - name: Stop and disable nova_api service @@ -408,6 +395,16 @@ outputs: file: path: /var/spool/cron/nova state: absent + external_upgrade_tasks: + - when: step|int == 1 + block: + - name: Online data migration for Nova + command: docker exec nova_api nova-manage db online_data_migrations + delegate_to: "{{ groups['nova_api'][0] }}" + become: true + tags: + - online_upgrade + - online_upgrade_nova fast_forward_upgrade_tasks: - when: - step|int == 0 diff --git a/releasenotes/notes/online-migration-external-upgrade-5093de6bd8993b5c.yaml b/releasenotes/notes/online-migration-external-upgrade-5093de6bd8993b5c.yaml new file mode 100644 index 0000000000..ea5f1495ea --- /dev/null +++ b/releasenotes/notes/online-migration-external-upgrade-5093de6bd8993b5c.yaml @@ -0,0 +1,16 @@ +--- +upgrade: + - | + The online part of the service upgrades (online data migrations) + is now run using:: + + openstack overcloud external-upgrade run --tags online_upgrade + + or per-service like:: + + openstack overcloud external-upgrade run --tags online_upgrade_nova + openstack overcloud external-upgrade run --tags online_upgrade_cinder + openstack overcloud external-upgrade run --tags online_upgrade_ironic + + Consult the upgrade documentation regarding the full upgrade + workflow.