Allows use of Mistral workflows during deployment steps

Introduces a general mechanism meant to allow for the execution
of workflows during the deployment steps.

Services can define workflow actions to be triggered during a step
in the newly added service_workflow_tasks section. The syntax is:

  service_workflow_tasks:
    step2:
      - name: my_action_name
        action: std.echo
        input:
          output: 'hello world'

Implements: blueprint tripleo-ceph-ansible
Depends-On: If02799e7457ca017cc119317dfb2db7198a3559f
Depends-On: Ibc5707f9f06266fe84ad1dd91dcb984157871d30
Change-Id: I36a642fbc2076ad9e4a10ffc56d6d16f3ed6f27a

docker/docker-steps.j2

@@ -21,6 +21,9 @@ parameters:
   servers:
     type: json
     description: Mapping of Role name e.g Controller to a list of servers
+  stack_name:
+    type: string
+    description: Name of the topmost stack
   role_data:
     type: json
     description: Mapping of Role name e.g Controller to the per-role data
@@ -36,6 +39,19 @@ parameters:
                  via parameter_defaults in the resource registry.
     type: json
 
+conditions:
+{% for step in range(1, deploy_steps_max) %}
+  WorkflowTasks_Step{{step}}_Enabled:
+    or:
+    {% for role in roles %}
+      - not:
+          equals:
+            - get_param: [role_data, {{role.name}}, service_workflow_tasks, step{{step}}]
+            - ''
+      - False
+    {% endfor %}
+{% endfor %}
+
 resources:
 
   # These utility tasks use docker-puppet.py to execute tasks via puppet
@@ -68,6 +84,47 @@ resources:
         - name: bootstrap_server_id
       config: {get_file: deploy-steps-playbook.yaml}
 
+{%- for step in range(1, deploy_steps_max) %}
+# BEGIN service_workflow_tasks handling
+  WorkflowTasks_Step{{step}}:
+    type: OS::Mistral::Workflow
+    condition: WorkflowTasks_Step{{step}}_Enabled
+    depends_on:
+    {% if step == 1 %}
+    {% for dep in roles %}
+      - {{dep.name}}PreConfig
+      - {{dep.name}}ArtifactsDeploy
+    {% endfor %}
+    {% else %}
+    {% for dep in roles %}
+      - {{dep.name}}Deployment_Step{{step -1}}
+    {% endfor %}
+    {% endif %}
+    properties:
+      name: {list_join: [".", ["tripleo", {get_param: stack_name}, "workflowtasks", "step{{step}}"]]}
+      type: direct
+      tasks:
+        yaql:
+          expression: $.data.where($ != '').select($.get('step{{step}}')).where($ != null).flatten()
+          data:
+          {% for role in roles %}
+            - get_param: [role_data, {{role.name}}, service_workflow_tasks]
+          {% endfor %}
+
+  WorkflowTasks_Step{{step}}_Execution:
+    type: OS::Mistral::ExternalResource
+    condition: WorkflowTasks_Step{{step}}_Enabled
+    depends_on: WorkflowTasks_Step{{step}}
+    properties:
+      actions:
+        CREATE:
+          workflow: { get_resource: WorkflowTasks_Step{{step}} }
+        UPDATE:
+          workflow: { get_resource: WorkflowTasks_Step{{step}} }
+      always_update: true
+# END service_workflow_tasks handling
+{% endfor %}
+
 {% for role in roles %}
   # Post deployment steps for all roles
   # A single config is re-applied with an incrementing step number
@@ -195,14 +252,23 @@ resources:
 
   {{role.name}}Deployment_Step{{step}}:
     type: OS::Heat::StructuredDeploymentGroup
-  {% if step == 1 %}
-    depends_on: [{{role.name}}PreConfig, {{role.name}}ArtifactsDeploy]
-  {% else %}
     depends_on:
-      {% for dep in roles %}
+      - WorkflowTasks_Step{{step}}_Execution
+    # TODO(gfidente): the following if/else condition
+    # replicates what is already defined for the
+    # WorkflowTasks_StepX resource and can be remove
+    # if https://bugs.launchpad.net/heat/+bug/1700569
+    # is fixed.
+    {% if step == 1 %}
+    {% for dep in roles %}
+      - {{dep.name}}PreConfig
+      - {{dep.name}}ArtifactsDeploy
+    {% endfor %}
+    {% else %}
+    {% for dep in roles %}
       - {{dep.name}}Deployment_Step{{step -1}}
-      {% endfor %}
-  {% endif %}
+    {% endfor %}
+    {% endif %}
     properties:
       name: {{role.name}}Deployment_Step{{step}}
       servers: {get_param: [servers, {{role.name}}]}

overcloud.j2.yaml

@@ -735,6 +735,7 @@ resources:
 {% for role in roles %}
         {{role.name}}: {get_attr: [{{role.name}}Servers, value]}
 {% endfor %}
+      stack_name: {get_param: 'OS::stack_name'}
       EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
       role_data:
 {% for role in roles %}

puppet/post.j2.yaml

@@ -8,7 +8,9 @@ parameters:
   servers:
     type: json
     description: Mapping of Role name e.g Controller to a list of servers
-
+  stack_name:
+    type: string
+    description: Name of the topmost stack
   role_data:
     type: json
     description: Mapping of Role name e.g Controller to the per-role data
@@ -24,5 +26,4 @@ parameters:
       Setting this to a unique value will re-run any deployment tasks which
       perform configuration on a Heat stack-update.
 
-resources:
 {% include 'puppet-steps.j2' %}

puppet/puppet-steps.j2

@@ -1,3 +1,19 @@
+{% set deploy_steps_max = 6 %}
+
+conditions:
+{% for step in range(1, deploy_steps_max) %}
+  WorkflowTasks_Step{{step}}_Enabled:
+    or:
+    {% for role in roles %}
+      - not:
+          equals:
+            - get_param: [role_data, {{role.name}}, service_workflow_tasks, step{{step}}]
+            - ''
+      - False
+    {% endfor %}
+{% endfor %}
+
+resources:
   # Post deployment steps for all roles
   # A single config is re-applied with an incrementing step number
 {% for role in roles %}
@@ -24,17 +40,26 @@
       StepConfig: {list_join: ["\n", {get_param: [role_data, {{role.name}}, step_config]}]}
 
   # Step through a series of configuration steps
-{% for step in range(1, 6) %}
+{% for step in range(1, deploy_steps_max) %}
   {{role.name}}Deployment_Step{{step}}:
     type: OS::Heat::StructuredDeploymentGroup
-  {% if step == 1 %}
-    depends_on: [{{role.name}}PreConfig, {{role.name}}ArtifactsDeploy]
-  {% else %}
     depends_on:
+      - WorkflowTasks_Step{{step}}_Execution
+    # TODO(gfidente): the following if/else condition
+    # replicates what is already defined for the
+    # WorkflowTasks_StepX resource and can be remove
+    # if https://bugs.launchpad.net/heat/+bug/1700569
+    # is fixed.
+    {% if step == 1 %}
+    {% for dep in roles %}
+      - {{dep.name}}PreConfig
+      - {{dep.name}}ArtifactsDeploy
+    {% endfor %}
+    {% else %}
     {% for dep in roles %}
       - {{dep.name}}Deployment_Step{{step -1}}
     {% endfor %}
-  {% endif %}
+    {% endif %}
     properties:
       name: {{role.name}}Deployment_Step{{step}}
       servers: {get_param: [servers, {{role.name}}]}
@@ -72,3 +97,44 @@
 
 
 {% endfor %}
+
+# BEGIN service_workflow_tasks handling
+{% for step in range(1, deploy_steps_max) %}
+  WorkflowTasks_Step{{step}}:
+    type: OS::Mistral::Workflow
+    condition: WorkflowTasks_Step{{step}}_Enabled
+    depends_on:
+    {% if step == 1 %}
+    {% for dep in roles %}
+      - {{dep.name}}PreConfig
+      - {{dep.name}}ArtifactsDeploy
+    {% endfor %}
+    {% else %}
+    {% for dep in roles %}
+      - {{dep.name}}Deployment_Step{{step -1}}
+    {% endfor %}
+    {% endif %}
+    properties:
+      name: {list_join: [".", ["tripleo", {get_param: stack_name}, "workflowtasks", "step{{step}}"]]}
+      type: direct
+      tasks:
+        yaql:
+          expression: $.data.where($ != '').select($.get('step{{step}}')).where($ != null).flatten()
+          data:
+          {% for role in roles %}
+            - get_param: [role_data, {{role.name}}, service_workflow_tasks]
+          {% endfor %}
+
+  WorkflowTasks_Step{{step}}_Execution:
+    type: OS::Mistral::ExternalResource
+    condition: WorkflowTasks_Step{{step}}_Enabled
+    depends_on: WorkflowTasks_Step{{step}}
+    properties:
+      actions:
+        CREATE:
+          workflow: { get_resource: WorkflowTasks_Step{{step}} }
+        UPDATE:
+          workflow: { get_resource: WorkflowTasks_Step{{step}} }
+      always_update: true
+{% endfor %}
+# END service_workflow_tasks handling

puppet/services/README.rst

@@ -95,6 +95,30 @@ are re-asserted when applying latter ones.
 
    5) Service activation (Pacemaker)
 
+It is also possible to use Mistral actions or workflows together with
+a deployment step, these are executed before the main configuration run.
+To describe actions or workflows from within a service use:
+
+  * service_workflow_tasks: One or more workflow task properties
+
+which expects a map where the key is the step and the value a list of
+dictionaries descrbing each a workflow task, for example::
+
+  service_workflow_tasks:
+    step2:
+      - name: echo
+        action: std.echo output=Hello
+    step3:
+      - name: external
+        workflow: my-pre-existing-workflow-name
+        input:
+          workflow_param1: value
+          workflow_param2: value
+
+The Heat guide for the `OS::Mistral::Workflow task property
+<https://docs.openstack.org/developer/heat/template_guide/openstack.html#OS::Mistral::Workflow-prop-tasks>`_
+has more details about the expected dictionary.
+
 Batch Upgrade Steps
 -------------------
 

releasenotes/notes/service_workflow_tasks-4da5830821b7154b.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    It is now possible to trigger Mistral workflows or workflow actions
+    before a deployment step is applied. This can be defined within the
+    scope of a service template and is described as a task property
+    for the Heat OS::Mistral::Workflow resource, for more details also
+    see the puppet/services/README.rst file.

services.yaml

@@ -116,6 +116,10 @@ outputs:
         yaql:
           expression: $.data.role_data.where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {})
           data: {role_data: {get_attr: [ServiceChain, role_data]}}
+      service_workflow_tasks:
+        yaql:
+          expression: $.data.role_data.where($ != null).select($.get('service_workflow_tasks')).where($ != null).reduce($1.mergeWith($2), {})
+          data: {role_data: {get_attr: [ServiceChain, role_data]}}
       step_config: {get_attr: [ServiceChain, role_data, step_config]}
       upgrade_tasks:
         yaql: