Introduce scale_tasks

Composable service templates can now define scale_tasks. They are
meant for scale down/up logic of services which need to be
stopped/started during the scaling procedure. All happens within a single
playbook and the down/up Ansible tags are required to differenciate them
during the run.

blueprint scale-down-tasks

Change-Id: I29835c2724b5de884e9825e084780c36837bff84
Co-Authored-By: Alex Schultz <aschultz@redhat.com>

common/deploy-steps.j2

@@ -24,6 +24,7 @@
 {% set fast_forward_upgrade_steps_max = 9 -%}
 {% set fast_forward_upgrade_prep_steps_max = 3 -%}
 {% set post_update_steps_max = 4 -%}
+{% set scale_steps_max = 1 -%}
 
 heat_template_version: rocky
 
@@ -148,6 +149,19 @@ resources:
             - get_param: [role_data, {{role.name}}, external_post_deploy_tasks]
           {%- endfor %}
 
+  ScaleTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          # processing from per-role unique tasks into globally unique tasks
+          expression: coalesce($.data, []).flatten().distinct()
+          data:
+          {%- for role in enabled_roles %}
+            - get_param: [role_data, {{role.name}}, scale_tasks]
+          {%- endfor %}
+
   ExternalUpdateTasks:
     type: OS::Heat::Value
     properties:
@@ -823,6 +837,89 @@ outputs:
               tags:
                 - external
                 - external_deploy_steps
+      scale_steps_tasks: {get_attr: [ScaleTasks, value]}
+      scale_playbook:
+        str_replace:
+          params:
+            DEPLOY_SOURCE_HOST: {get_param: deployment_source_hosts}
+            DEPLOY_TARGET_HOST: {get_param: deployment_target_hosts}
+            DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
+            BOOTSTRAP_SERVER_ID: {get_attr: [BootstrapServerId, value]}
+            ENABLE_DEBUG: {get_param: ConfigDebug}
+            ENABLE_PUPPET: {get_param: EnablePuppet}
+            DOCKER_PUPPET_DEBUG: {get_param: DockerPuppetDebug}
+            DOCKER_PUPPET_PROCESS_COUNT: {get_param: DockerPuppetProcessCount}
+            DOCKER_PUPPET_MOUNT_HOST_PUPPET: {get_param: DockerPuppetMountHostPuppet}
+            CONTAINER_CLI: {get_param: ContainerCli}
+            CONTAINER_LOG_STDOUT_PATH: {get_param: ContainerLogStdoutPath}
+          template: |
+            - hosts: DEPLOY_SOURCE_HOST
+              name: Gather facts from undercloud
+              gather_facts: yes
+              become: false
+              tags:
+                - always
+                - facts
+            # Collect the facts from the overcloud nodes but ignore unreachable
+            # nodes in the case of a dead node which needs to be part of the
+            # scale-down operation.
+            - hosts: DEPLOY_TARGET_HOST
+              name: Gather facts from overcloud
+              gather_facts: yes
+              ignore_unreachable: True
+              tags:
+                - always
+                - facts
+            - hosts: all
+              name: Load global variables
+              gather_facts: no
+              tasks:
+                - include_vars: global_vars.yaml
+              tags:
+                - always
+            - hosts: DEPLOY_TARGET_HOST
+              name: Scaling
+              gather_facts: no
+              any_errors_fatal: yes
+              become: false
+              vars:
+                bootstrap_server_id: BOOTSTRAP_SERVER_ID
+                deploy_identifier: DEPLOY_IDENTIFIER
+                enable_debug: ENABLE_DEBUG
+                enable_puppet: ENABLE_PUPPET
+                container_cli: CONTAINER_CLI
+                container_log_stdout_path: CONTAINER_LOG_STDOUT_PATH
+                docker_puppet_debug: DOCKER_PUPPET_DEBUG
+                docker_puppet_process_count: DOCKER_PUPPET_PROCESS_COUNT
+                docker_puppet_mount_host_puppet: DOCKER_PUPPET_MOUNT_HOST_PUPPET
+              tasks:
+                # Only run the scale tasks on the nodes that are alive.
+                # We expect the tasks to work on the nodes that are alive.
+                # If a task is allowed to fail, it needs to be configured at
+                # the task definition level but not here.
+                - include_tasks: scale_steps_tasks.yaml
+                  with_sequence: start=0 end={{scale_steps_max}}
+                  loop_control:
+                    loop_var: step
+                  tags: always
+                  # we use ansible_hostname to determine if the host is alive
+                  # or not.
+                  when: ansible_hostname is defined
+                # We don't want to run the scale tasks on dead nodes, to allow
+                # the operator to scale down the cloud no matter the state of
+                # the servers.
+                # However, we notify the operator if the node wasn't reachable.
+                # Using fail and not debug module to make it more visible
+                # in the logs.
+                - fail:
+                    msg: "Node is unreachable. No scale tasks will be run."
+                  ignore_errors: True
+                  tags: always
+                  # we use ansible_hostname to determine if the host is alive
+                  # or not.
+                  when: ansible_hostname is not defined
+              tags:
+                - scale
       fast_forward_upgrade_playbook:
         str_replace:
           params:

common/services/role.role.j2.yaml

@@ -175,6 +175,16 @@ resources:
           expression: coalesce($.data, []).where($ != null).select($.get('external_post_deploy_tasks')).where($ != null).flatten().distinct()
           data: {get_attr: [ServiceChain, role_data]}
 
+  ScaleTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: coalesce($.data, []).where($ != null).select($.get('scale_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
+
   ExternalUpdateTasks:
     type: OS::Heat::Value
     properties:
@@ -353,6 +363,7 @@ outputs:
       external_post_deploy_tasks: {get_attr: [ExternalPostDeployTasks, value]}
       external_update_tasks: {get_attr: [ExternalUpdateTasks, value]}
       external_upgrade_tasks: {get_attr: [ExternalUpgradeTasks, value]}
+      scale_tasks: {get_attr: [ScaleTasks, value]}
       fast_forward_upgrade_tasks: {get_attr: [FastForwardUpgradeTasks, value]}
       fast_forward_post_upgrade_tasks: {get_attr: [FastForwardPostUpgradeTasks, value]}
       pre_upgrade_rolling_tasks: {get_attr: [PreUpgradeRollingTasks, value]}

releasenotes/notes/scale_tasks-2042b294d074b37a.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Composable service templates can now define scale_tasks. They are
+    meant for scale down/up logic of services which need to be
+    stopped/started during the scaling procedure. All happens within a single
+    playbook and the down/up Ansible tags are required to differenciate them
+    during the run.

tools/yaml-validate.py

@@ -66,7 +66,7 @@ OPTIONAL_DOCKER_SECTIONS = ['container_puppet_tasks', 'upgrade_tasks',
                             'kolla_config', 'global_config_settings',
                             'external_deploy_tasks', 'external_post_deploy_tasks',
                             'container_config_scripts', 'step_config',
-                            'monitoring_subscription',
+                            'monitoring_subscription', 'scale_tasks',
                             'external_update_tasks', 'external_upgrade_tasks']
 # ansible tasks cannot be an empty dict or ansible is unhappy
 ANSIBLE_TASKS_SECTIONS = ['upgrade_tasks', 'pre_upgrade_rolling_tasks',