Migrate Ceph deployment to external_deploy_task

Uses external_deploy_tasks instead of workflow_tasks for the
deployment of Ceph, via ceph-ansible.

Initially, external_deploy_tasks are added alongside workflow_tasks so
that CI will pass. Once CI is updated to use config-download for these
jobs, we will remove the workflow_tasks.

Co-Authored-By: James Slagle <jslagle@redhat.com>
Co-Authored-By: John Fulton <fulton@redhat.com>
Implements: blueprint ceph-ansible-external-deploy-tasks
Change-Id: I4b88e97c38ff394023a92bec5631b3cd0f6e293e

docker/services/ceph-ansible/ceph-base.yaml

@@ -42,6 +42,11 @@ parameters:
     type: string
     default: '{}'
     description: json string containing per-node configuration map
+  DeploymentServerBlacklist:
+    default: []
+    type: comma_delimited_list
+    description: >
+      List of server hostnames to blacklist from any triggered deployments.
   CephAnsibleWorkflowName:
     type: string
     description: Name of the Mistral workflow to execute
@@ -150,9 +155,11 @@ parameters:
     description: image
     type: string
   CephAnsiblePlaybookVerbosity:
-    default: 0
+    default: 1
-    description: number of '-v', '-vv', etc. passed to ansible-playbook command (max 5)
+    description: The number of '-v', '-vv', etc. passed to ansible-playbook command
     type: number
+    constraints:
+    - range: { min: 1, max: 5 }
   CephAnsibleEnvironmentVariables:
     default: {}
     description: Mapping of Ansible environment variables to override defaults.
@@ -165,6 +172,11 @@ conditions:
       expression: $.data.split('/')[0].matches('(\.|:)')
   perform_upgrade:
     equals: [{get_param: StackUpdateType}, 'UPGRADE']
+  ceph_ansible_skip_tags_set:
+    not:
+      equals:
+        - {get_param: CephAnsibleSkipTags}
+        - ''
 
 resources:
   DockerImageUrlParts:
@@ -193,35 +205,12 @@ resources:
             expression: $.data.rightSplit(':', 1)[1]
             data: {get_param: DockerCephDaemonImage}
 
-outputs:
+  CephBaseAnsibleVars:
-  role_data:
+    type: OS::Heat::Value
-    description: Role data for the Ceph base service.
+    properties:
-    value:
+      type: json
-      service_name: ceph_base
+      value:
-      upgrade_tasks: []
+        vars:
-      puppet_config:
-        config_image: ''
-        config_volume: ''
-        step_config: ''
-      docker_config: {}
-      workflow_tasks:
-        step2:
-          - name: ceph_base_ansible_workflow
-            workflow: {get_param: CephAnsibleWorkflowName}
-            input:
-              ansible_skip_tags: {get_param: CephAnsibleSkipTags}
-              ceph_ansible_extra_vars: {get_param: CephAnsibleExtraConfig}
-              ceph_ansible_playbook:
-                if:
-                  - perform_upgrade
-                  - {get_param: CephAnsibleUpgradePlaybook}
-                  - {get_param: CephAnsiblePlaybook}
-              ansible_playbook_verbosity: {get_param: CephAnsiblePlaybookVerbosity}
-              node_data_lookup: {get_param: NodeDataLookup}
-              ansible_env_variables: {get_param: CephAnsibleEnvironmentVariables}
-      config_settings:
-        ceph_common_ansible_vars:
-          ireallymeanit: 'yes'
           fsid: { get_param: CephClusterFSID }
           cluster: { get_param: CephClusterName }
           docker: true
@@ -333,3 +322,139 @@ outputs:
             - {get_param: CephIPv6}
             - ipv6
             - ipv4
+
+outputs:
+  role_data:
+    description: Role data for the Ceph base service.
+    value:
+      service_name: ceph_base
+      upgrade_tasks: []
+      puppet_config:
+        config_image: ''
+        config_volume: ''
+        step_config: ''
+      docker_config: {}
+      workflow_tasks:
+        step2:
+          - name: ceph_base_ansible_workflow
+            workflow: {get_param: CephAnsibleWorkflowName}
+            input:
+              ansible_skip_tags: {get_param: CephAnsibleSkipTags}
+              ceph_ansible_extra_vars: {get_param: CephAnsibleExtraConfig}
+              ceph_ansible_playbook:
+                if:
+                  - perform_upgrade
+                  - {get_param: CephAnsibleUpgradePlaybook}
+                  - {get_param: CephAnsiblePlaybook}
+              ansible_playbook_verbosity: {get_param: CephAnsiblePlaybookVerbosity}
+              node_data_lookup: {get_param: NodeDataLookup}
+              ansible_env_variables: {get_param: CephAnsibleEnvironmentVariables}
+      config_settings:
+        ceph_common_ansible_vars:
+          map_merge:
+            - ireallymeanit: 'yes'
+            - {get_attr: [CephBaseAnsibleVars, value, vars]}
+      external_deploy_tasks:
+        - name: ceph_base_external_deploy_init
+          when: step == '1'
+          block:
+            - name: set blacklisted_hostnames
+              set_fact:
+                blacklisted_hostnames: {get_param: DeploymentServerBlacklist}
+            - name: create ceph-ansible temp dirs
+              file:
+                path: "{{item}}"
+                state: directory
+              with_items:
+                - "{{playbook_dir}}/ceph-ansible/group_vars"
+                - "{{playbook_dir}}/ceph-ansible/host_vars"
+                - "{{playbook_dir}}/ceph-ansible/fetch_dir"
+            - name: generate inventory
+              copy:
+                dest: "{{playbook_dir}}/ceph-ansible/inventory.yml"
+                content: |
+                  {%- set ceph_groups = ['mgr', 'mon', 'osd', 'mds', 'rgw', 'nfs', 'rbdmirror', 'client'] -%}
+                  {%- for ceph_group in ceph_groups -%}
+                  {%- if 'ceph_' ~ ceph_group in groups %}
+
+                  {{ ceph_group ~ 's:' }}
+                    hosts:
+                      {% for host in groups['ceph_' ~ ceph_group] -%}
+                      {%- if hostvars.raw_get(host)['ansible_hostname'] not in blacklisted_hostnames -%}
+                      {{ hostvars.raw_get(host)['ansible_hostname'] }}:
+                        ansible_user: {{ hostvars.raw_get(host)['ansible_ssh_user'] | default('root') }}
+                        ansible_host: {{ hostvars.raw_get(host)['ansible_host'] | default(host) }}
+                        ansible_become: true
+                      {% endif -%}
+                      {%- endfor -%}
+
+                  {%- endif -%}
+                  {%- endfor %}
+            - name: set ceph-ansible group vars all
+              set_fact:
+                ceph_ansible_group_vars_all: {get_attr: [CephBaseAnsibleVars, value, vars]}
+            - name: generate ceph-ansible group vars all
+              copy:
+                dest: "{{playbook_dir}}/ceph-ansible/group_vars/all.yml"
+                content: "{{ceph_ansible_group_vars_all|to_nice_yaml}}"
+            - name: set ceph-ansible extra vars
+              set_fact:
+                ceph_ansible_extra_vars:
+                  map_merge:
+                  - {get_param: CephAnsibleExtraConfig}
+                  - ireallymeanit: "yes"
+                    fetch_directory: "{{playbook_dir}}/ceph-ansible/fetch_dir"
+            - name: generate ceph-ansible extra vars
+              copy:
+                dest: "{{playbook_dir}}/ceph-ansible/extra_vars.yml"
+                content: "{{ceph_ansible_extra_vars|to_nice_yaml}}"
+            # TODO(gfidente): match the nodes machine uuids with
+            # hostnames and paste node specific configs into host vars
+            - name: generate collect nodes uuid playbook
+              copy:
+                dest: "{{playbook_dir}}/ceph-ansible/nodes_uuid_playbook.yml"
+                content: |
+                  - hosts: mgrs:mons:osds:mdss:rgws:nfss:rbdmirrors:clients
+                    gather_facts: no
+                    tasks:
+                      - name: collect machine id
+                        command: dmidecode -s system-uuid
+        - name: ceph_base_external_deploy_task
+          when: step == '2'
+          block:
+            - name: set ceph-ansible verbosity
+              set_fact:
+                ceph_ansible_playbook_verbosity: {get_param: CephAnsiblePlaybookVerbosity}
+            - name: set ceph-ansible command
+              set_fact:
+                ceph_ansible_command:
+                  list_join:
+                  - ' '
+                  - - ANSIBLE_ACTION_PLUGINS=/usr/share/ceph-ansible/plugins/actions/
+                    - ANSIBLE_ROLES_PATH=/usr/share/ceph-ansible/roles/
+                    - ANSIBLE_LOG_PATH="{{playbook_dir}}/ceph-ansible/ceph_ansible_command.log"
+                    - ANSIBLE_LIBRARY=/usr/share/ceph-ansible/library/
+                    - ANSIBLE_RETRY_FILES_ENABLED=False
+                    - ANSIBLE_SSH_RETRIES=3
+                    - ANSIBLE_HOST_KEY_CHECKING=False
+                    - DEFAULT_FORKS=25
+                    - yaql:
+                        data: {get_param: CephAnsibleEnvironmentVariables}
+                        expression: $.data.items().select($[0] + '=' + $[1]).join(' ')
+                    - ansible-playbook
+                    - '{% if ansible_ssh_private_key_file is defined %}--private-key {{ansible_ssh_private_key_file}}{% endif %}'
+                    - '-{%- for number in range(0, ceph_ansible_playbook_verbosity) -%}v{% endfor %}'
+                    - if:
+                      - ceph_ansible_skip_tags_set
+                      - list_join:
+                        - ' '
+                        - - '--skip-tags'
+                          - {get_param:  CephAnsibleSkipTags}
+                      - ''
+                    - '-i'
+                    - '{{playbook_dir}}/ceph-ansible/inventory.yml'
+                    - '--extra-vars'
+                    - '@{{playbook_dir}}/ceph-ansible/extra_vars.yml'
+            - name: run ceph-ansible
+              with_items: {get_param: CephAnsiblePlaybook}
+              shell: "{{ceph_ansible_command}} {{item}}"

docker/services/ceph-ansible/ceph-client.yaml

@@ -42,6 +42,13 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephClientAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars: {}
+
 outputs:
   role_data:
     description: Role data for the Ceph Client service.
@@ -54,4 +61,17 @@ outputs:
         step_config: ''
       docker_config: {}
       workflow_tasks: {get_attr: [CephBase, role_data, workflow_tasks]}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_client_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars clients
+                set_fact:
+                  ceph_ansible_group_vars_clients: {get_attr: [CephClientAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars clients
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/clients.yml"
+                  content: "{{ceph_ansible_group_vars_clients|to_nice_yaml}}"
       config_settings: {}

docker/services/ceph-ansible/ceph-external.yaml

@@ -46,6 +46,14 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephExternalAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars:
+          external_cluster_mon_ips: {get_param: CephExternalMonHost}
+
 outputs:
   role_data:
     description: Role data for the Ceph External service.
@@ -63,3 +71,17 @@ outputs:
           map_merge:
           - {get_attr: [CephBase, role_data, config_settings, ceph_common_ansible_vars]}
           - external_cluster_mon_ips: {get_param: CephExternalMonHost}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_external_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars clients
+                set_fact:
+                  ceph_ansible_group_vars_clients: {get_attr: [CephExternalAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars clients
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/clients.yml"
+                  content: "{{ceph_ansible_group_vars_clients|to_nice_yaml}}"
+      config_settings: {}

docker/services/ceph-ansible/ceph-mds.yaml

@@ -62,6 +62,19 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephMdsAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars:
+          cephfs_data: {get_param: ManilaCephFSDataPoolName}
+          cephfs_metadata: {get_param: ManilaCephFSMetadataPoolName}
+          cephfs: {get_param: ManilaCephFSNativeShareBackendName}
+          cephfs_pools:
+          - { name: {get_param: ManilaCephFSDataPoolName}, pgs: {get_param: ManilaCephFSDataPoolPGNum} }
+          - { name: {get_param: ManilaCephFSMetadataPoolName}, pgs: {get_param: ManilaCephFSMetadataPoolPGNum} }
+
 outputs:
   role_data:
     description: Role data for the Ceph Metadata service.
@@ -74,6 +87,19 @@ outputs:
         step_config: ''
       docker_config: {}
       workflow_tasks: {get_attr: [CephBase, role_data, workflow_tasks]}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_mds_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars mdss
+                set_fact:
+                  ceph_ansible_group_vars_mdss: {get_attr: [CephMdsAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars mdss
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/mdss.yml"
+                  content: "{{ceph_ansible_group_vars_mdss|to_nice_yaml}}"
       config_settings:
         map_merge:
         - tripleo.ceph_mds.firewall_rules:

docker/services/ceph-ansible/ceph-mgr.yaml

@@ -42,6 +42,14 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephMgrAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars:
+          ceph_mgr_docker_extra_env: '-e MGR_DASHBOARD=0'
+
 outputs:
   role_data:
     description: Role data for the Ceph Manager service.
@@ -54,6 +62,19 @@ outputs:
         step_config: ''
       docker_config: {}
       workflow_tasks: {get_attr: [CephBase, role_data, workflow_tasks]}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_mgr_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars mgrs
+                set_fact:
+                  ceph_ansible_group_vars_mgrs: {get_attr: [CephMgrAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars mgrs
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/mgrs.yml"
+                  content: "{{ceph_ansible_group_vars_mgrs|to_nice_yaml}}"
       config_settings:
         map_merge:
         - tripleo.ceph_mgr.firewall_rules:

docker/services/ceph-ansible/ceph-mon.yaml

@@ -59,6 +59,15 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephMonAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars:
+          monitor_secret: {get_param: CephMonKey}
+          admin_secret: {get_param: CephAdminKey}
+
 outputs:
   role_data:
     description: Role data for the Ceph Monitor service.
@@ -71,6 +80,19 @@ outputs:
         step_config: ''
       docker_config: {}
       workflow_tasks: {get_attr: [CephBase, role_data, workflow_tasks]}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_mon_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars mons
+                set_fact:
+                  ceph_ansible_group_vars_mons: {get_attr: [CephMonAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars mons
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/mons.yml"
+                  content: "{{ceph_ansible_group_vars_mons|to_nice_yaml}}"
       config_settings:
         map_merge:
         - tripleo.ceph_mon.firewall_rules:

docker/services/ceph-ansible/ceph-nfs.yaml

@@ -42,6 +42,20 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephNfsAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars:
+          ceph_nfs_bind_addr: {get_param: [EndpointMap, GaneshaInternal, host]}
+          ceph_nfs_enable_service: false
+          ceph_nfs_use_pacemaker: true
+          ceph_nfs_dynamic_exports: true
+          ceph_nfs_service_suffix: pacemaker
+          nfs_obj_gw: false
+          ceph_nfs_rados_backend: true
+
 outputs:
   role_data:
     description: Role data for the Ceph NFS Ganesha service.
@@ -55,6 +69,19 @@ outputs:
         step_config: ''
       # step_config seems to be ignored if docker_config is present
       #docker_config: {}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_nfs_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars nfss
+                set_fact:
+                  ceph_ansible_group_vars_nfss: {get_attr: [CephNfsAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars nfss
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/nfss.yml"
+                  content: "{{ceph_ansible_group_vars_nfss|to_nice_yaml}}"
       config_settings:
         map_merge:
         - tripleo.ceph_nfs.firewall_rules:

docker/services/ceph-ansible/ceph-osd.yaml

@@ -50,6 +50,16 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephOsdAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars:
+          map_merge:
+          - osd_objectstore: filestore
+          - {get_param: CephAnsibleDisksConfig}
+
 outputs:
   role_data:
     description: Role data for the Ceph OSD service.
@@ -66,6 +76,19 @@ outputs:
         step_config: ''
       docker_config: {}
       workflow_tasks: {get_attr: [CephBase, role_data, workflow_tasks]}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_osd_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars osds
+                set_fact:
+                  ceph_ansible_group_vars_osds: {get_attr: [CephOsdAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars osds
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/osds.yml"
+                  content: "{{ceph_ansible_group_vars_osds|to_nice_yaml}}"
       config_settings:
         map_merge:
         - tripleo.ceph_osd.firewall_rules:

docker/services/ceph-ansible/ceph-rbdmirror.yaml

@@ -65,6 +65,18 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephRbdMirrorAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars:
+          copy_admin_key: {get_param: CephRbdMirrorCopyAdminKey}
+          ceph_rbd_mirror_configure: {get_param: CephRbdMirrorConfigure}
+          ceph_rbd_mirror_pool: {get_param: CephRbdMirrorPool}
+          ceph_rbd_mirror_remote_cluster: {get_param: CephRbdMirrorRemoteCluster}
+          ceph_rbd_mirror_remote_user: {get_param: CephRbdMirrorRemoteUser}
+
 outputs:
   role_data:
     description: Role data for the Ceph RBD Mirror service.
@@ -77,6 +89,19 @@ outputs:
         step_config: ''
       docker_config: {}
       workflow_tasks: {get_attr: [CephBase, role_data, workflow_tasks]}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_rbdmirror_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars rbdmirrors
+                set_fact:
+                  ceph_ansible_group_vars_rbdmirrors: {get_attr: [CephRbdMirrorAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars rbdmirrors
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/rbdmirrors.yml"
+                  content: "{{ceph_ansible_group_vars_rbdmirrors|to_nice_yaml}}"
       config_settings:
         map_merge:
         - tripleo.ceph_rbdmirror.firewall_rules:

docker/services/ceph-ansible/ceph-rgw.yaml

@@ -50,6 +50,17 @@ resources:
       RoleName: {get_param: RoleName}
       RoleParameters: {get_param: RoleParameters}
 
+  CephRgwAnsibleVars:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        vars:
+          radosgw_keystone: true
+          radosgw_keystone_ssl: false
+          radosgw_address_block: {get_param: [ServiceData, net_cidr_map, {get_param: [ServiceNetMap, CephRgwNetwork]}]}
+          radosgw_civetweb_port: {get_param: [EndpointMap, CephRgwInternal, port]}
+
 outputs:
   role_data:
     description: Role data for the Ceph RadosGW service.
@@ -62,6 +73,19 @@ outputs:
         step_config: ''
       docker_config: {}
       workflow_tasks: {get_attr: [CephBase, role_data, workflow_tasks]}
+      external_deploy_tasks:
+        list_concat:
+        - {get_attr: [CephBase, role_data, external_deploy_tasks]}
+        - - name: ceph_rgw_external_deploy_init
+            when: step == '1'
+            block:
+              - name: set ceph-ansible group vars rgws
+                set_fact:
+                  ceph_ansible_group_vars_rgws: {get_attr: [CephRgwAnsibleVars, value, vars]}
+              - name: generate ceph-ansible group vars rgws
+                copy:
+                  dest: "{{playbook_dir}}/ceph-ansible/group_vars/rgws.yml"
+                  content: "{{ceph_ansible_group_vars_rgws|to_nice_yaml}}"
       config_settings:
         map_merge:
         - tripleo.ceph_rgw.firewall_rules: