Run online migrations via "external-upgrade run"

This will pull the online data migrations out of the upgrade
maintenance window and let them be performed after the main upgrade
phase while the cloud is already operational.

The online part of the service upgrades can be run using:

    openstack overcloud external-upgrade run --tags online_upgrade

or per-service like:

    openstack overcloud external-upgrade run --tags online_upgrade_nova
    openstack overcloud external-upgrade run --tags online_upgrade_cinder
    openstack overcloud external-upgrade run --tags online_upgrade_ironic

Change-Id: I35c8d9985df21b3084fba558687e1f408e5a0878
Closes-Bug: #1793332

docker/services/cinder-api.yaml

@@ -179,26 +179,6 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-        step_5:
-          cinder_api_online_migrations:
-            start_order: 2
-            image: *cinder_api_image
-            net: host
-            privileged: false
-            detach: false
-            user: root
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, volumes]}
-                -
-                  - /var/lib/config-data/cinder/etc/my.cnf.d/tripleo.cnf:/etc/my.cnf.d/tripleo.cnf:ro
-                  - /var/lib/config-data/cinder/etc/cinder/:/etc/cinder/:ro
-                  - /var/log/containers/cinder:/var/log/cinder
-                  - /var/log/containers/httpd/cinder-api:/var/log/httpd
-            command:
-              - '/usr/bin/bootstrap_host_exec'
-              - 'cinder_api'
-              - "su cinder -s /bin/bash -c 'cinder-manage db online_data_migrations'"
           cinder_api_cron:
             image: *cinder_api_image
             net: host
@@ -261,31 +241,35 @@ outputs:
             - name: Set fact cinder_api_enabled
               set_fact:
                 cinder_api_enabled: "{{ cinder_api_enabled_result.rc == 0 }}"
+            - name: Check if httpd service is running
+              command: systemctl is-active --quiet httpd
+              ignore_errors: True
+              register: httpd_running_result
+              when: httpd_running is undefined
+            - name: Set fact httpd_running
+              set_fact:
+                httpd_running: "{{ httpd_running_result.rc == 0 }}"
+              when: httpd_running is undefined
+            - name: check for cinder_api running under apache (post upgrade)
+              shell: "httpd -t -D DUMP_VHOSTS | grep -q cinder"
+              register: cinder_api_apache
+              ignore_errors: true
             - name: "PreUpgrade step0,validation: Check service openstack-cinder-api is running"
               shell: systemctl is-active --quiet openstack-cinder-api
               when: cinder_api_enabled|bool
               tags: validation
-            # NOTE: In puppet service variant, we ran release N online
+            - name: Ensure all online data migrations for Cinder have been applied
-            # migrations before upgrade to release N+1. In docker
+              command: docker exec cinder_api cinder-manage db online_data_migrations
-            # service variant, we run release N online migrations
-            # after upgrade to release N. This means that during
-            # switch from non-containerized to containerized, we need
-            # to run both of these.
-            - name: Online data migration for Cinder before switching to containers
               tags: pre-upgrade
               when:
                 - is_bootstrap_node|bool
-                - cinder_api_enabled|bool
+                - not (cinder_api_apache|bool and httpd_running|bool)
-              command: cinder-manage db online_data_migrations
+                - not cinder_api_enabled|bool
         - when: step|int == 2
           block:
             - name: Stop and disable cinder_api service (pre-upgrade not under httpd)
               service: name=openstack-cinder-api state=stopped enabled=no
               when: cinder_api_enabled|bool
-            - name: check for cinder_api running under apache (post upgrade)
-              shell: "httpd -t -D DUMP_VHOSTS | grep -q cinder"
-              register: cinder_api_apache
-              ignore_errors: true
             - name: Stop and disable cinder_api service
               service: name=httpd state=stopped enabled=no
               when: cinder_api_apache.rc == 0
@@ -302,6 +286,16 @@ outputs:
               package: name=httpd state=removed
               ignore_errors: True
               when: remove_httpd_package|bool
+      external_upgrade_tasks:
+        - when: step|int == 1
+          block:
+            - name: Online data migration for Cinder
+              command: docker exec cinder_api cinder-manage db online_data_migrations
+              delegate_to: "{{ groups['cinder_api'][0] }}"
+              become: true
+              tags:
+                - online_upgrade
+                - online_upgrade_cinder
       fast_forward_upgrade_tasks:
         - when:
             - step|int == 0

docker/services/ironic-api.yaml

@@ -138,22 +138,6 @@ outputs:
                   - /var/log/containers/httpd/ironic-api:/var/log/httpd
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-        step_5:
-          ironic_online_migrations:
-            start_order: 2
-            image: *ironic_api_image
-            net: host
-            privileged: false
-            detach: false
-            user: root
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, volumes]}
-                -
-                  - /var/lib/config-data/ironic_api/etc/ironic:/etc/ironic:ro
-                  - /var/log/containers/ironic:/var/log/ironic
-                  - /var/log/containers/httpd/ironic-api:/var/log/httpd
-            command: "/usr/bin/bootstrap_host_exec ironic_api su ironic -s /bin/bash -c 'ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations'"
       host_prep_tasks:
         - name: create persistent logs directory
           file:
@@ -213,18 +197,13 @@ outputs:
               when:
                 - ironic_httpd_enabled|bool
                 - httpd_running|bool
-            # NOTE: In puppet service variant, we ran release N online
+            - name: Ensure all online data migrations for Ironic have been applied
-            # migrations before upgrade to release N+1. In docker
+              command: docker exec ironic_api ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations
-            # service variant, we run release N online migrations
-            # after upgrade to release N. This means that during
-            # switch from non-containerized to containerized, we need
-            # to run both of these.
-            - name: Online data migration for Ironic before switching to containers
               tags: pre-upgrade
               when:
                 - is_bootstrap_node|bool
-                - (ironic_httpd_enabled|bool and httpd_running|bool) or ironic_api_enabled|bool
+                - not (ironic_httpd_enabled|bool and httpd_running|bool)
-              command: ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations
+                - not ironic_api_enabled|bool
         - when: step|int == 2
           block:
             - name: Stop and disable ironic_api service
@@ -245,6 +224,16 @@ outputs:
               package: name=openstack-ironic-api state=removed
               ignore_errors: True
               when: remove_ironic_api_package|bool
+      external_upgrade_tasks:
+        - when: step|int == 1
+          block:
+            - name: Online data migration for Ironic
+              command: docker exec ironic_api ironic-dbsync --config-file /etc/ironic/ironic.conf online_data_migrations
+              delegate_to: "{{ groups['ironic_api'][0] }}"
+              become: true
+              tags:
+                - online_upgrade
+                - online_upgrade_ironic
       fast_forward_upgrade_tasks:
         - when:
             - step|int == 0

docker/services/nova-api.yaml

@@ -320,14 +320,6 @@ outputs:
                   - ''
                   - - 'TRIPLEO_DEPLOY_IDENTIFIER='
                     - {get_param: DeployIdentifier}
-          nova_online_migrations:
-            start_order: 2
-            image: *nova_api_image
-            net: host
-            detach: false
-            volumes: *nova_api_bootstrap_volumes
-            user: root
-            command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage db online_data_migrations'"
       metadata_settings:
         get_attr: [NovaApiBase, role_data, metadata_settings]
       host_prep_tasks: {get_attr: [NovaApiLogging, host_prep_tasks]}
@@ -371,18 +363,13 @@ outputs:
               when:
                 - nova_api_httpd_enabled|bool
                 - httpd_running|bool
-            # NOTE: In puppet service variant, we ran release N online
+            - name: Ensure all online data migrations for Nova have been applied
-            # migrations before upgrade to release N+1. In docker
+              command: docker exec nova_api nova-manage db online_data_migrations
-            # service variant, we run release N online migrations
-            # after upgrade to release N. This means that during
-            # switch from non-containerized to containerized, we need
-            # to run both of these.
-            - name: Online data migration for Nova before switching to containers
               tags: pre-upgrade
               when:
                 - is_bootstrap_node|bool
-                - (nova_api_httpd_enabled|bool and httpd_running|bool) or nova_api_enabled|bool
+                - not (nova_api_httpd_enabled|bool and httpd_running|bool)
-              command: nova-manage db online_data_migrations
+                - not nova_api_enabled|bool
         - when: step|int == 2
           block:
             - name: Stop and disable nova_api service
@@ -408,6 +395,16 @@ outputs:
               file:
                 path: /var/spool/cron/nova
                 state: absent
+      external_upgrade_tasks:
+        - when: step|int == 1
+          block:
+            - name: Online data migration for Nova
+              command: docker exec nova_api nova-manage db online_data_migrations
+              delegate_to: "{{ groups['nova_api'][0] }}"
+              become: true
+              tags:
+                - online_upgrade
+                - online_upgrade_nova
       fast_forward_upgrade_tasks:
         - when:
             - step|int == 0

releasenotes/notes/online-migration-external-upgrade-5093de6bd8993b5c.yaml

@@ -0,0 +1,16 @@
+---
+upgrade:
+  - |
+    The online part of the service upgrades (online data migrations)
+    is now run using::
+
+        openstack overcloud external-upgrade run --tags online_upgrade
+
+    or per-service like::
+
+        openstack overcloud external-upgrade run --tags online_upgrade_nova
+        openstack overcloud external-upgrade run --tags online_upgrade_cinder
+        openstack overcloud external-upgrade run --tags online_upgrade_ironic
+
+    Consult the upgrade documentation regarding the full upgrade
+    workflow.