Update and upgrade tasks for services deployed via external deploy tasks

Composable service templates can now define external_update_tasks and
external_upgrade_tasks. They are meant for update/upgrade logic of
services deployed via external_deploy_tasks. The external update
playbook first executes external_update_tasks and then
external_deploy_tasks, the procedure for upgrades works
analogously. All happens within a single playbook, so variables or
fact overrides exported from the update/upgrade tasks will be
available to the deploy tasks during the update/upgrade procedure.

Partial-Bug: #1783949
Change-Id: Ib2474e8f69711cd6610a78884d5032ffd19ad249

common/deploy-steps.j2

@@ -16,8 +16,10 @@
 # primary role is: {{primary_role_name}}
 {% set deploy_steps_max = 6 -%}
 {% set update_steps_max = 6 -%}
+{% set external_update_steps_max = 1 -%}
 {% set pre_upgrade_rolling_steps_max = 1 -%}
 {% set upgrade_steps_max = 6 -%}
+{% set external_upgrade_steps_max = 1 -%}
 {% set post_upgrade_steps_max = 4 -%}
 {% set fast_forward_upgrade_steps_max = 9 -%}
 {% set fast_forward_upgrade_prep_steps_max = 3 -%}
@@ -170,6 +172,32 @@ resources:
             - get_param: [role_data, {{role.name}}, external_post_deploy_tasks]
           {%- endfor %}
 
+  ExternalUpdateTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          # processing from per-role unique tasks into globally unique tasks
+          expression: coalesce($.data, []).flatten().distinct()
+          data:
+          {%- for role in enabled_roles %}
+            - get_param: [role_data, {{role.name}}, external_update_tasks]
+          {%- endfor %}
+
+  ExternalUpgradeTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          # processing from per-role unique tasks into globally unique tasks
+          expression: coalesce($.data, []).flatten().distinct()
+          data:
+          {%- for role in enabled_roles %}
+            - get_param: [role_data, {{role.name}}, external_upgrade_tasks]
+          {%- endfor %}
+
 {%- for step in range(1, deploy_steps_max) %}
 # BEGIN workflow_tasks handling
   WorkflowTasks_Step{{step}}:
@@ -590,6 +618,58 @@ outputs:
                   with_sequence: start=0 end={{post_update_steps_max-1}}
                   loop_control:
                     loop_var: step
+      external_update_steps_tasks: {get_attr: [ExternalUpdateTasks, value]}
+      external_update_steps_playbook:
+        str_replace:
+          params:
+            DEPLOY_SOURCE_HOST: {get_param: deployment_source_hosts}
+            DEPLOY_TARGET_HOST: {get_param: deployment_target_hosts}
+          template: |
+            - hosts: DEPLOY_SOURCE_HOST
+              name: Gather facts from undercloud
+              gather_facts: yes
+              become: false
+              tags:
+                - always
+                - facts
+            # facts from overcloud may be needed for external installer inventory
+            - hosts: DEPLOY_TARGET_HOST
+              name: Gather facts from overcloud
+              gather_facts: yes
+            - hosts: all
+              name: Load global variables
+              gather_facts: no
+              tasks:
+                - include_vars: global_vars.yaml
+            - hosts: DEPLOY_SOURCE_HOST
+              name: External update steps
+              gather_facts: no
+              any_errors_fatal: yes
+              become: false
+              tasks:
+                - include: external_update_steps_tasks.yaml
+                  with_sequence: start=0 end={{external_update_steps_max-1}}
+                  loop_control:
+                    loop_var: step
+              tags:
+                - external
+                - external_update_steps
+            # putting both update and deploy tasks in the same
+            # playbook allows influencing the deploy tasks by
+            # variables "exported" from update tasks
+            - hosts: DEPLOY_SOURCE_HOST
+              name: External deploy steps
+              gather_facts: no
+              any_errors_fatal: yes
+              become: false
+              tasks:
+                - include: external_deploy_steps_tasks.yaml
+                  with_sequence: start=1 end={{deploy_steps_max-1}}
+                  loop_control:
+                    loop_var: step
+              tags:
+                - external
+                - external_deploy_steps
       pre_upgrade_rolling_steps_tasks: |
 {%- for role in roles %}
             - import_tasks: {{role.name}}/pre_upgrade_rolling_tasks.yaml
@@ -656,6 +736,58 @@ outputs:
                   with_sequence: start=0 end={{post_upgrade_steps_max-1}}
                   loop_control:
                     loop_var: step
+      external_upgrade_steps_tasks: {get_attr: [ExternalUpgradeTasks, value]}
+      external_upgrade_steps_playbook:
+        str_replace:
+          params:
+            DEPLOY_SOURCE_HOST: {get_param: deployment_source_hosts}
+            DEPLOY_TARGET_HOST: {get_param: deployment_target_hosts}
+          template: |
+            - hosts: DEPLOY_SOURCE_HOST
+              name: Gather facts from undercloud
+              gather_facts: yes
+              become: false
+              tags:
+                - always
+                - facts
+            # facts from overcloud may be needed for external installer inventory
+            - hosts: DEPLOY_TARGET_HOST
+              name: Gather facts from overcloud
+              gather_facts: yes
+            - hosts: all
+              name: Load global variables
+              gather_facts: no
+              tasks:
+                - include_vars: global_vars.yaml
+            - hosts: DEPLOY_SOURCE_HOST
+              name: External upgrade
+              gather_facts: no
+              any_errors_fatal: yes
+              become: false
+              tasks:
+                - include: external_upgrade_steps_tasks.yaml
+                  with_sequence: start=0 end={{external_upgrade_steps_max-1}}
+                  loop_control:
+                    loop_var: step
+              tags:
+                - external
+                - external_upgrade_steps
+            # putting both upgrade and deploy tasks in the same
+            # playbook allows influencing the deploy tasks by
+            # variables "exported" from upgrade tasks
+            - hosts: DEPLOY_SOURCE_HOST
+              name: External deploy steps
+              gather_facts: no
+              any_errors_fatal: yes
+              become: false
+              tasks:
+                - include: external_deploy_steps_tasks.yaml
+                  with_sequence: start=1 end={{deploy_steps_max-1}}
+                  loop_control:
+                    loop_var: step
+              tags:
+                - external
+                - external_deploy_steps
       fast_forward_upgrade_playbook:
         str_replace:
           params:

common/services.yaml

@@ -244,6 +244,26 @@ resources:
           expression: coalesce($.data, []).where($ != null).select($.get('external_post_deploy_tasks')).where($ != null).flatten().distinct()
           data: {get_attr: [ServiceChain, role_data]}
 
+  ExternalUpdateTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: coalesce($.data, []).where($ != null).select($.get('external_update_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
+
+  ExternalUpgradeTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: coalesce($.data, []).where($ != null).select($.get('external_upgrade_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
+
   FastForwardUpgradeTasks:
     type: OS::Heat::Value
     properties:
@@ -376,6 +396,8 @@ outputs:
       deploy_steps_tasks: {get_attr: [DeployStepsTasks, value]}
       external_deploy_tasks: {get_attr: [ExternalDeployTasks, value]}
       external_post_deploy_tasks: {get_attr: [ExternalPostDeployTasks, value]}
+      external_update_tasks: {get_attr: [ExternalUpdateTasks, value]}
+      external_upgrade_tasks: {get_attr: [ExternalUpgradeTasks, value]}
       fast_forward_upgrade_tasks: {get_attr: [FastForwardUpgradeTasks, value]}
       fast_forward_post_upgrade_tasks: {get_attr: [FastForwardPostUpgradeTasks, value]}
       pre_upgrade_rolling_tasks: {get_attr: [PreUpgradeRollingTasks, value]}

releasenotes/notes/external-update-upgrade-2d7bd96959ace08d.yaml

@@ -0,0 +1,11 @@
+---
+upgrade:
+  - |
+    Composable service templates can now define external_update_tasks
+    and external_upgrade_tasks. They are meant for update/upgrade
+    logic of services deployed via external_deploy_tasks. The external
+    update playbook first executes external_update_tasks and then
+    external_deploy_tasks, the procedure for upgrades works
+    analogously. All happens within a single playbook, so variables or
+    fact overrides exported from the update/upgrade tasks will be
+    available to the deploy tasks during the update/upgrade procedure.

tools/yaml-validate.py

@@ -70,7 +70,8 @@ OPTIONAL_DOCKER_SECTIONS = ['docker_puppet_tasks', 'upgrade_tasks',
                             'logging_source', 'logging_groups',
                             'external_deploy_tasks', 'external_post_deploy_tasks',
                             'docker_config_scripts', 'step_config',
-                            'monitoring_subscription']
+                            'monitoring_subscription',
+                            'external_update_tasks', 'external_upgrade_tasks']
 # ansible tasks cannot be an empty dict or ansible is unhappy
 ANSIBLE_TASKS_SECTIONS = ['upgrade_tasks', 'pre_upgrade_rolling_tasks',
                           'fast_forward_upgrade_tasks',