task-core basic framework

This change adds a new output to service definitions called
core_services which are used to define task-core services that will be
collected and exported during the config-download progress. These
services are mapped to TripleO roles and published into a task-core role
file in the config-download.

Related-Blueprint: unified-orchestration
Change-Id: I6954ecc92e740212a4502ac5fa8e53eeed22d043
Depends-On: https://review.opendev.org/c/openstack/tripleo-common/+/798721

ci/environments/scenario001-standalone.yaml

@@ -31,7 +31,10 @@ resource_registry:
   OS::TripleO::Services::SwiftProxy: OS::Heat::None
   OS::TripleO::Services::SwiftStorage: OS::Heat::None
   OS::TripleO::Services::SwiftRingBuilder: OS::Heat::None
-  OS::TripleO::Services::Horizon: OS::Heat::None
+  # NOTE(mwhahaha): cheat and use the horizon "service" for our test service
+  # since disable horizon. This allows us to not have to keep some test service
+  # in the defaults just for this case.
+  OS::TripleO::Services::Horizon: ../../deployment/tests/test-task-core.yaml
 
 parameter_defaults:
   NodeDataLookup:

common/deploy-steps.j2

@@ -46,6 +46,12 @@ parameters:
     description: >
       Host or hostgroup that consists of the target systems for the deployment.
       Defaults to all hosts in the current Heat stack if not set.
+  DeployHost:
+    default: {}
+    type: json
+    description: >
+      Deployment host related json. This data is used by tripleo-common
+      when generating deployment data.
   EndpointMap:
     default: {}
     description: Mapping of service endpoint -> protocol. Typically set
@@ -433,6 +439,24 @@ outputs:
       container_puppet_script: {get_file: ./container-puppet.sh}
       generate-config-tasks: {get_file: generate-config-tasks.yaml}
       host-container-puppet-tasks: {get_file: host-container-puppet-tasks.yaml}
+      task_core_data:
+        deployment_host: {get_param: DeployHost}
+        deployment_services:
+            deployment-state: {get_file: ./services/deployment-state.yaml}
+            deployment-bootstrap: {get_file: ./services/deployment-bootstrap.yaml}
+            deployment-network: {get_file: ./services/deployment-network.yaml}
+            deployment-host-prep: {get_file: ./services/deployment-host-prep.yaml}
+            deployment-legacy-deploy: {get_file: ./services/deployment-legacy-deploy.yaml}
+            deployment-post-deploy: {get_file: ./services/deployment-post-deploy.yaml}
+        deployment_roles:
+          deployer:
+            services:
+              - deployment-state
+              - deployment-bootstrap
+              - deployment-network
+              - deployment-host-prep
+              - deployment-legacy-deploy
+              - deployment-post-deploy
       deploy_steps_0_playbook:
 {% block deploy_steps_str_replace_params %}
         str_replace:

common/services/deployment-bootstrap.yaml

@@ -0,0 +1,71 @@
+id: deployment-bootstrap
+type: service
+version: 1.0.0
+tasks:
+  - id: start
+    driver: print
+    provides:
+      - deployment-bootstrap.start
+    requires:
+      - deployment-state.bootstrap-start
+    message: "DEPLOYMENT: bootstrap start"
+
+  - id: deploy-steps-0
+    driver: ansible_runner
+    provides:
+      - deployment-bootstrap.deploy-steps-0
+    requires:
+      - deployment-bootstrap.start
+    playbook: deploy_steps_0_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: selinux
+    driver: ansible_runner
+    provides:
+      - deployment-bootstrap.selinux
+    requires:
+      - deployment-bootstrap.deploy-steps-0
+    playbook: deploy_steps_selinux_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: common-roles
+    driver: ansible_runner
+    provides:
+      - deployment-bootstrap.common-roles
+    requires:
+      - deployment-bootstrap.selinux
+    playbook: deploy_steps_common_roles_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: deploy-steps-0-tasks
+    driver: ansible_runner
+    provides:
+      - deployment-bootstrap.deploy-steps-0-tasks
+    requires:
+      - deployment-bootstrap.common-roles
+    playbook: deploy_steps_0_tasks_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: pre-network
+    driver: ansible_runner
+    provides:
+      - deployment-bootstrap.pre-network
+    requires:
+      - deployment-bootstrap.deploy-steps-0-tasks
+    playbook: deploy_steps_pre_network_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: end
+    driver: print
+    provides:
+      - deployment-bootstrap.end
+    requires:
+      - deployment-bootstrap.pre-network
+    needed-by:
+      - deployment-state.bootstrap-end
+    message: "DEPLOYMENT: bootstrap end"

common/services/deployment-host-prep.yaml

@@ -0,0 +1,52 @@
+id: deployment-host-prep
+type: service
+version: 1.0.0
+tasks:
+  - id: start
+    driver: print
+    provides:
+      - deployment-host-prep.start
+    requires:
+      - deployment-state.host-prep-start
+    message: "DEPLOYMENT: host prep start"
+
+  - id: pre-deployment
+    driver: ansible_runner
+    provides:
+      - deployment-host-prep.pre-deployment
+    requires:
+      - deployment-host-prep.start
+    playbook: deploy_steps_pre_deployment_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: host-prep
+    driver: ansible_runner
+    provides:
+      - deployment-host-prep.host-prep
+    requires:
+      - deployment-host-prep.pre-deployment
+    playbook: deploy_steps_host_prep_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: container-setup
+    driver: ansible_runner
+    provides:
+      - deployment-host-prep.container-setup
+    requires:
+      - deployment-host-prep.host-prep
+    playbook: deploy_steps_container_setup_tasks_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: end
+    driver: print
+    provides:
+      - deployment-host-prep.end
+    requires:
+      - deployment-host-prep.container-setup
+    needed-by:
+      - deployment-state.host-prep-end
+    message: "DEPLOYMENT: host prep end"
+

common/services/deployment-legacy-deploy.yaml

@@ -0,0 +1,31 @@
+id: deployment-legacy-deploy
+type: service
+version: 1.0.0
+tasks:
+  - id: start
+    driver: print
+    provides:
+      - deployment-legacy-deploy.start
+    requires:
+      - deployment-state.legacy-deploy-start
+    message: "DEPLOYMENT: Legacy Deployment start"
+
+  - id: deployment
+    driver: ansible_runner
+    provides:
+      - deployment-legacy-deploy.deployment
+    requires:
+      - deployment-legacy-deploy.start
+    playbook: deploy_steps_external_deployment_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: end
+    driver: print
+    provides:
+      - deployment-legacy-deploy.end
+    requires:
+      - deployment-legacy-deploy.deployment
+    needed-by:
+      - deployment-state.legacy-deploy-end
+    message: "DEPLOYMENT: Legacy Deployment end"

common/services/deployment-network.yaml

@@ -0,0 +1,41 @@
+id: deployment-network
+type: service
+version: 1.0.0
+tasks:
+  - id: start
+    driver: print
+    provides:
+      - deployment-network.start
+    requires:
+      - deployment-state.network-start
+    message: "DEPLOYMENT: network configuration start"
+
+  - id: network-deployment
+    driver: ansible_runner
+    provides:
+      - deployment-network.deployment
+    requires:
+      - deployment-network.start
+    playbook: deploy_steps_network_deployments_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: network-validation
+    driver: ansible_runner
+    provides:
+      - deployment-network.validation
+    requires:
+      - deployment-network.deployment
+    playbook: deploy_steps_network_validation_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: end
+    driver: print
+    provides:
+      - deployment-network.end
+    requires:
+      - deployment-network.validation
+    needed-by:
+      - deployment-state.network-end
+    message: "DEPLOYMENT: network configuration"

common/services/deployment-post-deploy.yaml

@@ -0,0 +1,42 @@
+id: deployment-post-deploy
+type: service
+version: 1.0.0
+tasks:
+  - id: start
+    driver: print
+    provides:
+      - deployment-post-deploy.start
+    requires:
+      - deployment-state.post-deploy-start
+    message: "DEPLOYMENT: post deploy start"
+
+  - id: deployment
+    driver: ansible_runner
+    provides:
+      - deployment-post-deploy.deployment
+    requires:
+      - deployment-post-deploy.start
+    playbook: deploy_steps_post_deployment_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: tasks
+    driver: ansible_runner
+    provides:
+      - deployment-post-deploy.tasks
+    requires:
+      - deployment-post-deploy.deployment
+    playbook: deploy_steps_post_deploy_tasks_playbook.yaml
+    runner_options:
+      extravars: global_vars.yaml
+
+  - id: end
+    driver: print
+    provides:
+      - deployment-post-deploy.end
+    requires:
+      - deployment-post-deploy.tasks
+    needed-by:
+      - deployment-state.post-deploy-end
+    message: "DEPLOYMENT: post deploy start"
+

common/services/deployment-state.yaml

@@ -0,0 +1,104 @@
+id: deployment-state
+type: service
+version: 1.0.0
+tasks:
+  - id: start
+    driver: print
+    provides:
+      - deployment-state.start
+    message: "STATE: start"
+
+  - id: bootstrap-start
+    driver: print
+    provides:
+      - deployment-state.bootstrap-start
+    requires:
+      - deployment-state.start
+    message: "STATE: bootstrap start"
+
+  - id: bootstrap-end
+    driver: print
+    provides:
+      - deployment-state.bootstrap-end
+    requires:
+      - deployment-state.bootstrap-start
+      - deployment-bootstrap.end
+    message: "STATE: bootstrap end"
+
+  - id: network-start
+    driver: print
+    provides:
+      - deployment-state.network-start
+    requires:
+      - deployment-state.bootstrap-end
+    message: "STATE: network start"
+
+  - id: network-end
+    driver: print
+    provides:
+      - deployment-state.network-end
+    requires:
+      - deployment-state.network-start
+      - deployment-network.end
+    message: "STATE: network end"
+
+  - id: host-prep-start
+    driver: print
+    provides:
+      - deployment-state.host-prep-start
+    requires:
+      - deployment-state.network-end
+    message: "STATE: host prep start"
+
+  - id: host-prep-end
+    driver: print
+    provides:
+      - deployment-state.host-prep-end
+    requires:
+      - deployment-state.host-prep-start
+      - deployment-host-prep.end
+    message: "STATE: host prep end"
+
+  - id: legacy-deploy-start
+    driver: print
+    provides:
+      - deployment-state.legacy-deploy-start
+    requires:
+      - deployment-state.host-prep-end
+    message: "STATE: legacy deploy start"
+
+  - id: legacy-deploy-end
+    driver: print
+    provides:
+      - deployment-state.legacy-deploy-end
+    requires:
+      - deployment-state.legacy-deploy-start
+      - deployment-legacy-deploy.start
+    message: "STATE: legacy deploy end"
+
+  - id: post-deploy-start
+    driver: print
+    provides:
+      - deployment-state.post-deploy-start
+    requires:
+      - deployment-state.legacy-deploy-end
+    message: "STATE: post deploy start"
+
+  - id: post-deploy-end
+    driver: print
+    provides:
+      - deployment-state.post-deploy-end
+    requires:
+      - deployment-state.post-deploy-start
+    message: "STATE: post deploy end"
+
+  - id: end
+    driver: print
+    provides:
+      - deployment-state.end
+    requires:
+      - deployment-state.post-deploy-end
+      - deployment-post-deploy.end
+    message: "STATE: end"
+
+

common/services/role.role.j2.yaml

@@ -131,6 +131,16 @@ resources:
             expression: list(coalesce($.data.role_data, []).where($ != null).select($.get('keystone_resources')).where($ != null)).flatten()
             data: {role_data: {get_attr: [ServiceChain, role_data]}}
 
+  CoreServices:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        map_merge:
+          yaql:
+            expression: list(coalesce($.data.role_data, []).where($ != null).select($.get('core_services')).where($ != null))
+            data: {role_data: {get_attr: [ServiceChain, role_data]}}
+
   ServiceConfigSettings:
     type: OS::Heat::Value
     properties:
@@ -347,6 +357,7 @@ outputs:
     description: Combined Role data for this set of services.
     value:
       service_names: {get_attr: [ServiceNames, value]}
+      core_services: {get_attr: [CoreServices, value]}
       monitoring_subscriptions: {get_attr: [MonitoringSubscriptionsConfig, value]}
       config_settings:
         yaql:

deployment/README.rst

@@ -128,6 +128,9 @@ are available for containerized services.
        foo: bar
    This will set the hiera key 'foo' on all roles where haproxy is included.
 
+ * core_services: A dictionary of task-core service definitions that are
+   exported during the config-download process.
+
 Deployment steps
 ----------------
 Similar to baremetal containers are brought up in a stepwise manner.

deployment/tests/test-task-core.yaml

@@ -0,0 +1,68 @@
+heat_template_version: wallaby
+
+description: >
+  Configures a test service with task core tasks
+
+parameters:
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  ServiceData:
+    default: {}
+    description: Dictionary packing service data
+    type: json
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry. Use
+                 parameter_merge_strategies to merge it with the defaults.
+    type: json
+  RoleName:
+    default: ''
+    description: Role name on which the service is applied
+    type: string
+  RoleParameters:
+    default: {}
+    description: Parameters specific to the role
+    type: json
+
+outputs:
+  role_data:
+    description: Role data for a test task core service
+    value:
+      service_name: test_task_core
+      core_services:
+        # key should match service_name
+        test_task_core:
+          version: 0.0.1
+          tasks:
+            - id: print
+              driver: print
+              message: "message from test_task_core"
+            - id: setup
+              action: init
+              driver: service
+              provides:
+                - test_task_core.init
+              jobs:
+                - echo: "test task core init"
+            - id: run
+              action: run
+              driver: service
+              provides:
+                - test_task_core.run
+              requires:
+                - test_task_core.init
+              jobs:
+                - echo: "test task core run"
+            - id: finalize
+              action: finalize
+              driver: service
+              provides:
+                - test_task_core.finalize
+              requires:
+                - test_task_core.run
+              jobs:
+                - echo: "test task core finalize"