Rework gnocchi-upgrade to run in a separate upgrade step

gnocchi when configured with swift will require keystone
to be available to authenticate to migrate to v3. At this
step keystone is not available and gnocchi upgrade fails
with auth error. Instead start apache in step 3, start
apache first and then run gnocchi upgrade in a separate
step and let upgrade happen here.

Closes-Bug: #1634897

Change-Id: I22d02528420e4456f84b80905a7b3a80653fa7b0

extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh

@@ -64,6 +64,5 @@ if [[ -n $(is_bootstrap_node) ]]; then
     nova-manage db sync
     nova-manage api_db sync
     nova-manage db online_data_migrations
-    gnocchi-upgrade
     sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head
 fi

extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh

@@ -9,18 +9,9 @@ check_resource redis started 600
 start_or_enable_service openstack-cinder-volume
 check_resource openstack-cinder-volume started 600
 
+# start httpd so keystone is available for gnocchi
+# upgrade to run.
+systemctl start httpd
 
 # Swift isn't controled by pacemaker
 systemctl_swift start
-
-# We need to start the systemd services we explicitely stopped at step _1.sh
-# FIXME: Should we let puppet during the convergence step do the service enabling or
-# should we add it here?
-services=$(services_to_migrate)
-if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then
-    services=${services%%openstack-sahara*}
-fi
-for service in $services; do
-    manage_systemd_service start "${service%%-clone}"
-    check_resource_systemd "${service%%-clone}" started 600
-done

extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+
+if [[ -n $(is_bootstrap_node) ]]; then
+  # run gnocchi upgrade
+  gnocchi-upgrade
+fi

extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eu
+
+# We need to start the systemd services we explicitely stopped at step _1.sh
+# FIXME: Should we let puppet during the convergence step do the service enabling or
+# should we add it here?
+services=$(services_to_migrate)
+if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then
+    services=${services%%openstack-sahara*}
+fi
+for service in $services; do
+    manage_systemd_service start "${service%%-clone}"
+    check_resource_systemd "${service%%-clone}" started 600
+done

extraconfig/tasks/major_upgrade_pacemaker.yaml

@@ -132,13 +132,7 @@ resources:
       config:
         list_join:
         - ''
-        - - str_replace:
-              template: |
-                #!/bin/bash
-                keep_sahara_services_on_upgrade='KEEP_SAHARA_SERVICES_ON_UPGRADE'
-              params:
-                KEEP_SAHARA_SERVICES_ON_UPGRADE: {get_param: KeepSaharaServicesOnUpgrade}
-          - get_file: pacemaker_common_functions.sh
+        - - get_file: pacemaker_common_functions.sh
           - get_file: major_upgrade_pacemaker_migrations.sh
           - get_file: major_upgrade_controller_pacemaker_3.sh
 
@@ -150,3 +144,46 @@ resources:
       config: {get_resource: ControllerPacemakerUpgradeConfig_Step3}
       input_values: {get_param: input_values}
 
+  ControllerPacemakerUpgradeConfig_Step4:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_pacemaker_migrations.sh
+          - get_file: major_upgrade_controller_pacemaker_4.sh
+
+  ControllerPacemakerUpgradeDeployment_Step4:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: ControllerPacemakerUpgradeDeployment_Step3
+    properties:
+      servers:  {get_param: [servers, Controller]}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step4}
+      input_values: {get_param: input_values}
+
+  ControllerPacemakerUpgradeConfig_Step5:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - str_replace:
+              template: |
+                #!/bin/bash
+                keep_sahara_services_on_upgrade='KEEP_SAHARA_SERVICES_ON_UPGRADE'
+              params:
+                KEEP_SAHARA_SERVICES_ON_UPGRADE: {get_param: KeepSaharaServicesOnUpgrade}
+          - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_pacemaker_migrations.sh
+          - get_file: major_upgrade_controller_pacemaker_5.sh
+
+  ControllerPacemakerUpgradeDeployment_Step5:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: ControllerPacemakerUpgradeDeployment_Step4
+    properties:
+      servers:  {get_param: [servers, Controller]}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step5}
+      input_values: {get_param: input_values}