Add external deployment tasks executed on undercloud

Services can define external_deploy_tasks, which are meant to be
executed on the undercloud node. They are step-based as the other
Ansible tasks we have, and they get executed during each deployment
step before the puppet and docker tasks.

These tasks can be used to perform complex actions from the
undercloud, such as executing nested installers like kubespray or
ceph-ansible. This should allow deploying overcloud with a single
Ansible playbook, and without creating Ansible->Mistral->Ansible loop.

Implements: blueprint ansible-config-download
Change-Id: I3dcafb96f5cea5fdcebe2b2012b61a38b0568834
Depends-On: I8491540edf78711f3229eabeda22a17cd55e99c8

common/deploy-steps.j2

@@ -97,6 +97,19 @@ resources:
           params:
             _TASKS: {get_file: deploy-steps-tasks.yaml}
 
+  ExternalDeployTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          # processing from per-role unique tasks into globally unique tasks
+          expression: coalesce($.data, []).flatten().distinct()
+          data:
+          {%- for role in enabled_roles %}
+            - get_param: [role_data, {{role.name}}, external_deploy_tasks]
+          {%- endfor %}
+
 {%- for step in range(1, deploy_steps_max) %}
 # BEGIN workflow_tasks handling
   WorkflowTasks_Step{{step}}:
@@ -354,10 +367,21 @@ outputs:
                 - include: {{role.name}}/host_prep_tasks.yaml
                   when: role_name == '{{role.name}}'
 {%- endfor %}
+{%- for step in range(1,deploy_steps_max) %}
+            - hosts: undercloud
+              name: Undercloud deployment step {{step}}
+              vars:
+                step: '{{step}}'
+              tasks:
+                - include: external_deploy_steps_tasks.yaml
+            - hosts: overcloud
+              name: Overcloud deployment step {{step}}
+              vars:
+                bootstrap_server_id: BOOTSTRAP_SERVER_ID
+                step: '{{step}}'
+              tasks:
                 - include: deploy_steps_tasks.yaml
-                  with_sequence: count={{deploy_steps_max-1}}
-                  loop_control:
-                    loop_var: step
+{%- endfor %}
             - hosts: overcloud
               name: Server Post Deployments
               tasks:
@@ -368,6 +392,7 @@ outputs:
                   when: role_name == '{{role.name}}'
                   with_items: "{{ '{{' }} {{role.name}}_post_deployments {{ '}}' }}"
 {%- endfor %}
+      external_deploy_steps_tasks: {get_attr: [ExternalDeployTasks, value]}
       update_steps_tasks: |
 {%- for role in roles %}
             - include: {{role.name}}/update_tasks.yaml

common/services.yaml

@@ -183,6 +183,16 @@ resources:
           expression: coalesce($.data.role_data, []).where($ != null).select($.get('workflow_tasks')).where($ != null).reduce($1.mergeWith($2), {})
           data: {role_data: {get_attr: [ServiceChain, role_data]}}
 
+  ExternalDeployTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: coalesce($.data, []).where($ != null).select($.get('external_deploy_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
+
   UpgradeTasks:
     type: OS::Heat::Value
     properties:
@@ -272,6 +282,7 @@ outputs:
       service_config_settings: {get_attr: [ServiceConfigSettings, value]}
       workflow_tasks: {get_attr: [WorkflowTasks, value]}
       step_config: {get_attr: [PuppetStepConfig, value]}
+      external_deploy_tasks: {get_attr: [ExternalDeployTasks, value]}
       upgrade_tasks: {get_attr: [UpgradeTasks, value]}
       post_upgrade_tasks: {get_attr: [PostUpgradeTasks, value]}
       update_tasks: {get_attr: [UpdateTasks, value]}

tools/yaml-validate.py

@@ -48,7 +48,7 @@ OPTIONAL_DOCKER_SECTIONS = ['docker_puppet_tasks', 'upgrade_tasks',
                             'service_config_settings', 'host_prep_tasks',
                             'metadata_settings', 'kolla_config',
                             'global_config_settings', 'logging_source',
-                            'logging_groups']
+                            'logging_groups', 'external_deploy_tasks']
 REQUIRED_DOCKER_PUPPET_CONFIG_SECTIONS = ['config_volume', 'step_config',
                                           'config_image']
 OPTIONAL_DOCKER_PUPPET_CONFIG_SECTIONS = [ 'puppet_tags', 'volumes' ]