Introduce ceph_spec_bootstrap module

The ceph_spec_bootstrap Ansible module uses information
about the enabled services, roles, and deployed hosts to
determine what Ceph services should run on what hosts and
generate a valid Ceph spec file.

This allows the desired end state defined in TripleO to be
translated into an end state defined in Ceph orchestrator.
The intention is to use this module when bootstrapping Ceph.

This change removes files/ceph_spec.yaml, which is a spec
file for TripleO standalone deployments in THT scenario001.
This file will now be created dynamically by the new module.

Also, fix spelling error in ceph-admin-user-playbook.yml.

Change-Id: I74cb373f1dc94f8a5ef12c0545cfbc0c49abd69b

.ansible-lint

@@ -19,6 +19,7 @@ mock_modules:
     - ceph_fs
     - ceph_pool
     - ceph_mkspec
+    - ceph_spec_bootstrap
     - config_template
     - container_startup_config
     - lvm2_physical_devices_facts

doc/source/modules/modules-ceph_spec_bootstrap.rst

@@ -0,0 +1,14 @@
+============================
+Module - ceph_spec_bootstrap
+============================
+
+
+This module provides for the following ansible plugin:
+
+    * ceph_spec_bootstrap
+
+
+.. ansibleautoplugin::
+   :module: tripleo_ansible/ansible_plugins/modules/ceph_spec_bootstrap.py
+   :documentation: true
+   :examples: true

tripleo_ansible/ansible_plugins/modules/ceph_spec_bootstrap.py

@@ -0,0 +1,486 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Create Ceph Orchestrator specification file based on TripleO parameters"""
+
+import os
+import re
+import yaml
+
+from ansible.module_utils.basic import AnsibleModule
+try:
+    from ansible.module_utils import ceph_spec
+except ImportError:
+    from tripleo_ansible.ansible_plugins.module_utils import ceph_spec
+
+
+ANSIBLE_METADATA = {
+    'metadata_version': '0.1',
+    'status': ['preview'],
+    'supported_by': 'community'
+}
+
+DOCUMENTATION = '''
+---
+module: ceph_spec_bootstrap module
+short_description: Create Ceph Orchestrator specification file based on TripleO parameters
+description:
+    - "The ceph_spec_bootstrap module uses information from both the composed services in TripleO roles and the deployed hosts file ('openstack overcloud node provision' output), or just the inventory file (tripleo-ansible-inventory output) to determine what Ceph services should run on what hosts and generate a valid Ceph spec. This allows the desired end state defined in TripleO to be translated into an end state defined in Ceph orchestrator. The intention is to use this module when bootstraping a new Ceph cluster."
+options:
+    deployed_metalsmith:
+        description: The absolute path to a file like deployed_metal.yaml, as genereated by 'openstack overcloud node provision --output deployed_metal.yaml'. This file is used to map which ceph_service_types map to which deployed hosts. Use this option if you have deployed servers with metalsmith but do not yet have an inventory genereated from the overcloud in Heat. Either tripleo_ansible_inventory xor deployed_metalsmith must be used (not both).
+        required: False
+        type: str
+    tripleo_ansible_inventory:
+        description: The absolute path to an Ansible inventory genereated by running the tripleo-ansible-inventory command. This file is used to map which ceph_service_types map to which deployed hosts. Use this option if you already have an inventory genereated from the overcloud in Heat. Either tripleo_ansible_inventory xor deployed_metalsmith must be used (not both).
+        required: False
+        type: str
+    new_ceph_spec:
+        description: The absolute path to a new file which will be created by the module and contain the resultant Ceph specification. If not provided, defaults to /home/stack/ceph_spec.yaml.
+        required: False
+        type: str
+    ceph_service_types:
+        description: List of Ceph services being deployed on overcloud. All service names must be a valid service_type as described in the Ceph Orchestrator CLI service spec documentation. If not provided, defaults to ['mon', 'mgr', 'osd'], which are presently the only supported service types this module supports.
+        required: False
+        type: list
+    tripleo_roles:
+        description: The absolute path to the TripleO roles file. Only necessary if deployed_metalsmith is used. If not provided then defaults to /usr/share/openstack-tripleo-heat-templates/roles_data.yaml. This file is used to map which ceph_service_types map to which roles. E.g. all roles with OS::TripleO::Services::CephOSD will get the Ceph service_type 'osd'. This paramter is ignored if tripleo_ansible_inventory is used.
+        required: False
+        type: str
+    osd_spec:
+        description: A valid osd service specification. If not passed defaults to using all available data devices (data_devices all true).
+        required: False
+        type: dict
+    fqdn:
+        description: When true, the "hostname" and "hosts" in the generated Ceph spec will have their fully qualified domain name. This paramter defaults to false and only has an effect when tripleo_ansible_inventory is used.
+        required: False
+        type: bool
+author:
+    - John Fulton (fultonj)
+'''
+
+EXAMPLES = '''
+- name: make spec from 'openstack overcloud node provision' output
+  ceph_spec_bootstrap:
+    new_ceph_spec: "{{ playbook_dir }}/ceph_spec.yaml"
+    deployed_metalsmith: ~/overcloud-baremetal-deployed.yaml
+
+- name: make spec from tripleo-ansible-inventory output
+  ceph_spec_bootstrap:
+    new_ceph_spec: "{{ playbook_dir }}/ceph_spec.yaml"
+    tripleo_ansible_inventory: ~/config-download/overcloud/tripleo-ansible-inventory.yaml
+
+- name: make spec from inventory with FQDNs and custom osd_spec
+  ceph_spec_bootstrap:
+    new_ceph_spec: "{{ playbook_dir }}/ceph_spec.yaml"
+    tripleo_ansible_inventory: ~/config-download/overcloud/tripleo-ansible-inventory.yaml
+    fqdn: true
+    osd_spec:
+      data_devices:
+        paths:
+          - /dev/ceph_vg/ceph_lv_data
+
+- name: make spec with only Ceph mons and managers
+  ceph_spec_bootstrap:
+    new_ceph_spec: "{{ playbook_dir }}/ceph_spec.yaml"
+    deployed_metalsmith: ~/overcloud-baremetal-deployed.yaml
+    ceph_service_types:
+      - mon
+      - mgr
+
+- name: make spec with composed roles/ HDDs for data/ SSDs for db
+  ceph_spec_bootstrap:
+    new_ceph_spec: "{{ playbook_dir }}/ceph_spec.yaml"
+    deployed_metalsmith: ~/overcloud-baremetal-deployed.yaml
+    tripleo_roles: ~/templates/custom_roles_data.yaml
+    osd_spec:
+      data_devices:
+        rotational: 1
+      db_devices:
+        rotational: 0
+'''
+
+RETURN = '''
+'''
+
+# Map tripleo services to ceph spec service_types
+SERVICE_MAP = {
+    'CephMon': ['mon'],
+    'CephMgr': ['mgr'],
+    'CephOSD': ['osd']
+}
+
+# Support for the following are not yet available
+#   'CephMds': ['mds'],
+#   'CephRbdMirror': ['rbd-mirror'],
+#   'CephRgw': ['rgw'],
+#   'CephGrafana': ['alertmanager', 'grafana', 'node-exporter'],
+
+
+def get_inventory_hosts_to_ips(inventory, roles, fqdn=False):
+    """Return a map of hostnames to IP addresses, e.g.
+         {'oc0-ceph-0': '192.168.24.13',
+          'oc0-compute-0': '192.168.24.21',
+          'oc0-controller-0': '192.168.24.23',
+          'oc0-controller-1': '192.168.24.15',
+          'oc0-controller-2': '192.168.24.7'}
+    Uses ansible inventory as source
+    """
+    hosts_to_ips = {}
+    for key in inventory:
+        if key in roles:
+            for host in inventory[key]['hosts']:
+                ip = inventory[key]['hosts'][host]['ansible_host']
+                if fqdn:
+                    hostname = inventory[key]['hosts'][host]['canonical_hostname']
+                else:
+                    hostname = host
+                hosts_to_ips[hostname] = ip
+    return hosts_to_ips
+
+
+def get_deployed_hosts_to_ips(metalsmith_data_file):
+    """Return a map of hostnames to IP addresses, e.g.
+         {'oc0-ceph-0': '192.168.24.13',
+          'oc0-compute-0': '192.168.24.21',
+          'oc0-controller-0': '192.168.24.23',
+          'oc0-controller-1': '192.168.24.15',
+          'oc0-controller-2': '192.168.24.7'}
+    Uses output of metalsmith deployed hosts file as source
+    """
+    hosts_to_ips = {}
+    with open(metalsmith_data_file, 'r') as stream:
+        try:
+            metal = yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            print(exc)
+        try:
+            port_map = metal['parameter_defaults']['DeployedServerPortMap']
+            for host, host_map in port_map.items():
+                try:
+                    ip = host_map['fixed_ips'][0]['ip_address']
+                except Exception:
+                    raise RuntimeError(
+                        'The DeployedServerPortMap is missing the first '
+                        'fixed_ip in the data file: {metalsmith_data_file}'.format(
+                            metalsmith_data_file=metalsmith_data_file))
+                hosts_to_ips[host.replace('-ctlplane', '')] = ip
+        except Exception:
+            raise RuntimeError(
+                'The DeployedServerPortMap is not defined in '
+                'data file: {metalsmith_data_file}'.format(
+                metalsmith_data_file=metalsmith_data_file))
+    return hosts_to_ips
+
+
+def get_inventory_roles_to_hosts(inventory, roles, fqdn=False):
+    """Return a map of roles to host lists, e.g.
+         roles_to_hosts['CephStorage'] = ['oc0-ceph-0', 'oc0-ceph-1']
+         roles_to_hosts['Controller'] = ['oc0-controller-0']
+         roles_to_hosts['Compute'] = ['oc0-compute-0']
+    Uses ansible inventory as source
+    """
+    roles_to_hosts = {}
+    for key in inventory:
+        if key in roles:
+            roles_to_hosts[key] = []
+            for host in inventory[key]['hosts']:
+                if fqdn:
+                    hostname = inventory[key]['hosts'][host]['canonical_hostname']
+                else:
+                    hostname = host
+                roles_to_hosts[key].append(hostname)
+    return roles_to_hosts
+
+
+def get_deployed_roles_to_hosts(metalsmith_data_file, roles):
+    """Return a map of roles to host lists, e.g.
+         roles_to_hosts['CephStorage'] = ['oc0-ceph-0', 'oc0-ceph-1']
+         roles_to_hosts['Controller'] = ['oc0-controller-0']
+         roles_to_hosts['Compute'] = ['oc0-compute-0']
+    Uses output of metalsmith deployed hosts file as source
+    """
+    roles_to_hosts = {}
+    with open(metalsmith_data_file, 'r') as stream:
+        try:
+            metal = yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            print(exc)
+        try:
+            name_map = metal['parameter_defaults']['HostnameMap']
+            for role in roles:
+                for item in metal['parameter_defaults']:
+                    if item == role + 'HostnameFormat':
+                        host_fmt = metal['parameter_defaults'][item]
+                        pat = host_fmt.replace('%stackname%', '.*').replace('-%index%', '')
+                        reg = re.compile(pat)
+                        matching_hosts = []
+                        for host in name_map:
+                            if reg.match(host):
+                                matching_hosts.append(name_map[host])
+                roles_to_hosts[role] = matching_hosts
+        except Exception:
+            raise RuntimeError(
+                'The expected HostnameMap and RoleHostnameFormat are '
+                'not defined in data file: {metalsmith_data_file}'.format(
+                metalsmith_data_file=metalsmith_data_file))
+    return roles_to_hosts
+
+
+def get_roles_to_svcs_from_inventory(inventory):
+    """Return a map of map of TripleO Roles to TripleO Ceph Services, e.g.
+         {'CephStorage': ['CephOSD'],
+          'Controller': ['CephMgr', 'CephMon']}
+       Uses inventory file as source
+    """
+    # This approach is backwards but lets the larger program stay consistent
+    # and not require the roles file when the inventory is provided. The method
+    # of inventory is only used to deploy ceph during overcloud (not before).
+    roles_to_services = {}
+    inverse_service_map = {}
+    ceph_services = []
+    for tripleo_name, ceph_list in SERVICE_MAP.items():
+        for ceph_name in ceph_list:
+            ceph_services.append(ceph_name)
+            inverse_service_map[ceph_name] = tripleo_name
+    for key in inventory:
+        key_rename = key.replace('ceph_', '')
+        if key_rename in ceph_services:
+            for role in inventory[key]['children'].keys():
+                if role in roles_to_services.keys():
+                    roles_to_services[role].append(inverse_service_map[key_rename])
+                else:
+                    roles_to_services[role] = [inverse_service_map[key_rename]]
+    return roles_to_services
+
+
+def get_roles_to_svcs_from_roles(roles_file):
+    """Return a map of map of TripleO Roles to TripleO Ceph Services, e.g.
+         {'Compute': [],
+          'CephStorage': ['CephOSD'],
+          'Controller': ['CephMgr', 'CephMon']}
+       Uses roles file as source
+    """
+    roles_to_services = {}
+    with open(roles_file, 'r') as stream:
+        try:
+            roles = yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            print(exc)
+        try:
+            for role in roles:
+                svcs = []
+                for svc in role['ServicesDefault']:
+                    svc_short = svc.replace('OS::TripleO::Services::', '')
+                    if svc_short in SERVICE_MAP.keys():
+                        svcs.append(svc_short)
+                    roles_to_services[role['name']] = svcs
+        except Exception:
+            raise RuntimeError(
+                'Unable to extract the name or ServicesDefault list from '
+                'data file: {roles_file}'.format(roles_file=roles_file))
+    return roles_to_services
+
+
+def get_label_map(hosts_to_ips, roles_to_svcs, roles_to_hosts, ceph_service_types):
+    """Return a map of hostname to list of ceph service to run on that host, e.g.
+         label_map['oc0-ceph-0'] = ['osd']
+         label_map['oc0-controller-0'] = ['mon', 'mgr']
+    """
+    label_map = {}
+    for host in hosts_to_ips:
+        label_map[host] = []
+        for role, host_list in roles_to_hosts.items():
+            if host in host_list:
+                for tripleo_svc in roles_to_svcs[role]:
+                    for potential_ceph_svc in SERVICE_MAP[tripleo_svc]:
+                        if potential_ceph_svc in ceph_service_types:
+                            label_map[host].append(potential_ceph_svc)
+    return label_map
+
+
+def get_specs(hosts_to_ips, label_map, ceph_service_types, osd_spec={}):
+    """Build specs from hosts map, label_map, and ceph_service_types list
+       Create a ceph_spec object for each host or service
+       Returns a list of dictionaries.
+    """
+    specs = []
+    # Create host entries
+    for host, ip in hosts_to_ips.items():
+        if len(label_map[host]) > 0:
+            spec = ceph_spec.CephHostSpec('host', ip, host, label_map[host])
+            specs.append(spec.make_daemon_spec())
+
+    # Create service entries for supported services in SERVICE_MAP
+    labels = []
+    placement_pattern = ''
+    spec_dict = {}
+    for svc in ceph_service_types:
+        host_list = []
+        for host, label_list in label_map.items():
+            if svc in label_list:
+                host_list.append(host)
+        if svc in ['mon', 'mgr']:
+            d = ceph_spec.CephDaemonSpec(svc, svc, svc, host_list,
+                                         placement_pattern,
+                                         spec_dict, labels)
+        if svc in ['osd']:
+            if osd_spec == {}:
+                # default to all devices
+                osd_spec = {
+                    'data_devices': {
+                        'all': True
+                    }
+                }
+            d = ceph_spec.CephDaemonSpec(svc, 'default_drive_group',
+                                         'osd.default_drive_group',
+                                         host_list, placement_pattern,
+                                         spec_dict, labels, **osd_spec)
+        specs.append(d.make_daemon_spec())
+    return specs
+
+
+def render(specs, output):
+    """Write a multiline yaml file from a list of dicts
+    """
+    open(output, 'w').close()  # reset file
+    for spec in specs:
+        with open(output, 'a') as f:
+            f.write('---\n')
+            f.write(yaml.dump(spec))
+
+
+def flatten(t):
+    """Merge a list of lists into a single list
+    """
+    return [item for sublist in t for item in sublist]
+
+
+def main():
+    """Main method of Ansible module
+    """
+    result = dict(
+        changed=False,
+        msg='',
+        specs=[]
+    )
+    module = AnsibleModule(
+        argument_spec=yaml.safe_load(DOCUMENTATION)['options'],
+        supports_check_mode=True,
+    )
+    # Set payload defaults
+    result['failed'] = False
+    specs = []
+    errors = []
+
+    # Collect inputs
+    deployed_metalsmith = module.params.get('deployed_metalsmith')
+    tripleo_ansible_inventory = module.params.get('tripleo_ansible_inventory')
+    new_ceph_spec = module.params.get('new_ceph_spec')
+    ceph_service_types = module.params.get('ceph_service_types')
+    tripleo_roles = module.params.get('tripleo_roles')
+    osd_spec = module.params.get('osd_spec')
+    fqdn = module.params.get('fqdn')
+
+    # Set defaults
+    if ceph_service_types is None:
+        ceph_service_types = ['mon', 'mgr', 'osd']
+    if new_ceph_spec is None:
+        new_ceph_spec = "/home/stack/ceph_spec.yaml"
+    if tripleo_roles is None:
+        tripleo_roles = "/usr/share/openstack-tripleo-heat-templates/roles_data.yaml"
+    if osd_spec is None:
+        osd_spec = {}
+    if fqdn is None:
+        fqdn = False
+
+    # Validate inputs
+    # 0. Are they using metalsmith xor an inventory as their method?
+    method = ""
+    required_files = []
+    if deployed_metalsmith is None and tripleo_ansible_inventory is not None:
+        method = 'inventory'
+        required_files.append(tripleo_ansible_inventory)
+    elif deployed_metalsmith is not None and tripleo_ansible_inventory is None:
+        method = 'metal'
+        required_files.append(deployed_metalsmith)
+        required_files.append(tripleo_roles)
+    else:
+        error = "You must provide either the "
+        error += "tripleo_ansible_inventory or deployed_metalsmith "
+        error += "parameter (but not both)."
+        errors.append(error)
+        result['failed'] = True
+    # 1. The required files must all be an existing path to a file
+    for fpath in required_files:
+        if not os.path.isfile(fpath):
+            error = str(fpath) + " is not a valid file."
+            errors.append(error)
+            result['failed'] = True
+    # 2. The directory for the spec file must be an existing path
+    fpath = os.path.dirname(new_ceph_spec)
+    if not os.path.isdir(fpath):
+        error = str(fpath) + " is not a valid directory."
+        errors.append(error)
+        result['failed'] = True
+    # 3. argument_spec already ensures osd_spec is a dictionary
+    # 4. Must be one of the ceph_spec.ALLOWED_DAEMONS used in the SERVICE_MAP
+    supported_services = flatten(SERVICE_MAP.values())
+    for service_type in ceph_service_types:
+        if service_type not in supported_services:
+            error = "'" + str(service_type) + "' must be one of "
+            error += str(supported_services)
+            errors.append(error)
+            result['failed'] = True
+    # 5. fqdn is only supported for the inventory method
+    if method != 'inventory' and fqdn:
+        error = "The fqdn option may only be true when using tripleo_ansible_inventory"
+        errors.append(error)
+        result['failed'] = True
+
+    if not result['failed']:
+        # Build data structures to map roles/services/hosts/labels
+        if method == 'metal':
+            roles_to_svcs = get_roles_to_svcs_from_roles(tripleo_roles)
+            roles_to_hosts = get_deployed_roles_to_hosts(deployed_metalsmith,
+                                                         roles_to_svcs.keys())
+            hosts_to_ips = get_deployed_hosts_to_ips(deployed_metalsmith)
+        elif method == 'inventory':
+            with open(tripleo_ansible_inventory, 'r') as stream:
+                inventory = yaml.safe_load(stream)
+            roles_to_svcs = get_roles_to_svcs_from_inventory(inventory)
+            roles_to_hosts = get_inventory_roles_to_hosts(inventory,
+                                                          roles_to_svcs.keys(),
+                                                          fqdn)
+            hosts_to_ips = get_inventory_hosts_to_ips(inventory,
+                                                      roles_to_svcs.keys(),
+                                                      fqdn)
+        # regardless of how we built our maps, assign the correct labels
+        label_map = get_label_map(hosts_to_ips, roles_to_svcs,
+                                  roles_to_hosts, ceph_service_types)
+        # Build specs as list of ceph_spec objects from data structures
+        specs = get_specs(hosts_to_ips, label_map, ceph_service_types, osd_spec)
+        # Render specs list to file
+        render(specs, new_ceph_spec)
+
+    # Set payloads
+    result['msg'] = "  ".join(errors)
+    result['specs'] = specs
+
+    # exit and pass the key/value results
+    module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()

tripleo_ansible/playbooks/ceph-admin-user-playbook.yml

@@ -30,12 +30,12 @@
       stat:
         path: "{{ public_key }}"
       register: public_key_stat
-    - name: create private key if it does not eixst
+    - name: create private key if it does not exist
       shell: "ssh-keygen -t rsa -q -N '' -f {{ private_key }}"
       no_log: true
       when:
         - not private_key_stat.stat.exists
-    - name: create public key if it does not eixst
+    - name: create public key if it does not exist
       shell: "ssh-keygen -y -f {{ private_key }} > {{ public_key }}"
       when:
         - not public_key_stat.stat.exists

tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml

@@ -14,10 +14,14 @@ tripleo_cephadm_container_options: "--net=host --ipc=host"
 tripleo_cephadm_admin_keyring: "{{ tripleo_cephadm_config_home }}/{{ tripleo_cephadm_cluster }}.client.admin.keyring"
 tripleo_cephadm_conf: "{{ tripleo_cephadm_config_home }}/{{ tripleo_cephadm_cluster }}.conf"
 tripleo_cephadm_bootstrap_conf: "/home/{{ tripleo_cephadm_ssh_user }}/bootstrap_{{ tripleo_cephadm_cluster }}.conf"
+# path on ansible host (i.e. undercloud) of the ceph spec
+tripleo_cephadm_spec_ansible_host: "{{ playbook_dir }}/ceph_spec.yaml"
+# path on bootstrap node of ceph spec (scp'd from above var)
 tripleo_cephadm_spec: "/home/{{ tripleo_cephadm_ssh_user }}/specs/ceph_spec.yaml"
-tripleo_cephadm_spec_home: "/home/{{ tripleo_cephadm_ssh_user }}/specs"
+# path in container on bootstrap node of spec (podman -v'd from above var)
 tripleo_cephadm_container_spec: /home/ceph_spec.yaml
-tripleo_cephadm_spec_ansible_host: "{{ role_path }}/files/ceph_spec.yaml"
+# path of other ceph specs podman -v mounted into running container
+tripleo_cephadm_spec_home: "/home/{{ tripleo_cephadm_ssh_user }}/specs"
 tripleo_cephadm_bootstrap_files:
   - "/home/{{ tripleo_cephadm_ssh_user }}/.ssh/id_rsa"
   - "/home/{{ tripleo_cephadm_ssh_user }}/.ssh/id_rsa.pub"

tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml

@@ -1,24 +0,0 @@
----
-service_type: host
-addr: standalone.localdomain
-hostname: standalone.localdomain
----
-service_type: mon
-placement:
-  hosts:
-    - standalone.localdomain
----
-service_type: mgr
-service_name: mgr
-placement:
-  hosts:
-    - standalone.localdomain
----
-service_type: osd
-service_id: standalone_drive_group
-placement:
-  hosts:
-    - standalone.localdomain
-data_devices:
-  paths:
-    - /dev/ceph_vg/ceph_lv_data

tripleo_ansible/roles/tripleo_cephadm/molecule/default/converge.yml

@@ -31,7 +31,7 @@
         tasks_from: bootstrap
 
     - name: Mock ceph_mon_dump command
-      shell: "cat mock_ceph_mon_dump.json"
+      shell: "cat mock/mock_ceph_mon_dump.json"
       register: ceph_mon_mock_dump
       delegate_to: localhost
 

tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock/mock_deployed_metal.yaml

@@ -0,0 +1,40 @@
+---
+parameter_defaults:
+  CephStorageCount: 3
+  CephStorageHostnameFormat: '%stackname%-cephstorage-%index%'
+  ComputeCount: 1
+  ComputeHostnameFormat: '%stackname%-novacompute-%index%'
+  ControllerCount: 3
+  ControllerHostnameFormat: '%stackname%-controller-%index%'
+  DeployedServerPortMap:
+    oc0-ceph-0-ctlplane:
+      fixed_ips:
+        - ip_address: 192.168.24.13
+    oc0-ceph-1-ctlplane:
+      fixed_ips:
+        - ip_address: 192.168.24.11
+    oc0-ceph-2-ctlplane:
+      fixed_ips:
+        - ip_address: 192.168.24.14
+    oc0-compute-0-ctlplane:
+      fixed_ips:
+        - ip_address: 192.168.24.21
+    oc0-controller-0-ctlplane:
+      fixed_ips:
+        - ip_address: 192.168.24.23
+    oc0-controller-1-ctlplane:
+      fixed_ips:
+        - ip_address: 192.168.24.15
+    oc0-controller-2-ctlplane:
+      fixed_ips:
+        - ip_address: 192.168.24.7
+  HostnameMap:
+    oc0-cephstorage-0: oc0-ceph-0
+    oc0-cephstorage-1: oc0-ceph-1
+    oc0-cephstorage-2: oc0-ceph-2
+    oc0-controller-0: oc0-controller-0
+    oc0-controller-1: oc0-controller-1
+    oc0-controller-2: oc0-controller-2
+    oc0-novacompute-0: oc0-compute-0
+resource_registry:
+  OS::TripleO::DeployedServer::ControlPlanePort: /usr/share/openstack-tripleo-heat-templates/deployed-server/deployed-neutron-port.yaml

tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock/mock_inventory.yml

@@ -0,0 +1,33 @@
+---
+Standalone:
+  hosts:
+    standalone:
+      ansible_host: 192.168.24.1
+      canonical_hostname: standalone.localdomain
+      ctlplane_hostname: standalone.ctlplane.localdomain
+      ctlplane_ip: 192.168.24.1
+ceph_osd:
+  children:
+    Standalone: {}
+  vars:
+    ansible_ssh_user: root
+ceph_mgr:
+  children:
+    Standalone: {}
+  vars:
+    ansible_ssh_user: root
+ceph_client:
+  children:
+    Standalone: {}
+  vars:
+    ansible_ssh_user: root
+ceph_mon:
+  children:
+    Standalone: {}
+  vars:
+    ansible_ssh_user: root
+nova_libvirt:
+  children:
+    Standalone: {}
+  vars:
+    ansible_ssh_user: root

tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock/mock_overcloud_roles.yaml

@@ -0,0 +1,27 @@
+---
+- name: Controller
+  HostnameFormatDefault: '%stackname%-controller-%index%'
+  ServicesDefault:
+    - OS::TripleO::Services::Aide
+    - OS::TripleO::Services::CephClient
+    - OS::TripleO::Services::CephExternal
+    - OS::TripleO::Services::CephGrafana
+    - OS::TripleO::Services::CephMds
+    - OS::TripleO::Services::CephMgr
+    - OS::TripleO::Services::CephMon
+    - OS::TripleO::Services::CephRbdMirror
+    - OS::TripleO::Services::CephRgw
+    - OS::TripleO::Services::ManilaBackendCephFs
+    - OS::TripleO::Services::Zaqar
+- name: Compute
+  HostnameFormatDefault: '%stackname%-novacompute-%index%'
+  ServicesDefault:
+    - OS::TripleO::Services::Aide
+    - OS::TripleO::Services::CephClient
+    - OS::TripleO::Services::CephExternal
+    - OS::TripleO::Services::OVNMetadataAgent
+- name: CephStorage
+  ServicesDefault:
+    - OS::TripleO::Services::Aide
+    - OS::TripleO::Services::CephOSD
+    - OS::TripleO::Services::CertmongerUser

tripleo_ansible/roles/tripleo_cephadm/molecule/default/prepare.yml

@@ -38,3 +38,9 @@
         group: ceph-admin
         groups: wheel
         generate_ssh_key: true
+    - name: Create ceph_spec
+      ceph_spec_bootstrap:
+        new_ceph_spec: "{{ playbook_dir }}/ceph_spec.yaml"
+        deployed_metalsmith: mock/mock_deployed_metal.yaml
+        tripleo_roles: mock/mock_overcloud_roles.yaml
+      delegate_to: localhost

tripleo_ansible/roles/tripleo_cephadm/molecule/default/tasks/verify.yml

@@ -88,3 +88,50 @@
   vars:
     osd_profile: 'profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images'
     ips: '[v2:172.16.11.241:3300/0,v1:172.16.11.241:6789/0],[v2:172.16.11.176:3300/0,v1:172.16.11.176:6789/0],[v2:172.16.11.82:3300/0,v1:172.16.11.82:6789/0]'
+
+- name: Read spec file genereated from ceph_spec_bootstrap module
+  shell: "cat {{ tripleo_cephadm_spec_ansible_host }}"
+  register: cat_ceph_spec
+  delegate_to: localhost
+
+- name: Assert expected values about hosts entries
+  assert:
+    that:
+      - item.hostname is match("oc0-(controller|ceph)-(0|1|2)")
+      - item.addr is match("192.168.24.[0-9]{1,2}")
+      - (item.labels | join(' ')) is match ("mgr|mon|osd")
+  when:
+    - item.service_type == 'host'
+  loop: "{{ cat_ceph_spec.stdout | from_yaml_all | list }}"
+
+- name: Assert expected values about mon and mgr daemon entries
+  assert:
+    that:
+      - item.placement.hosts == expected_hosts
+      - item.service_id == item.service_name
+  when:
+    - item.service_type == 'mon' or item.service_type == 'mgr'
+  loop: "{{ cat_ceph_spec.stdout | from_yaml_all | list }}"
+  vars:
+    expected_hosts:
+      - oc0-controller-0
+      - oc0-controller-1
+      - oc0-controller-2
+
+- name: Assert expected values about osd daemon entries
+  assert:
+    that:
+      - item.placement.hosts == expected_hosts
+      - item.service_id == 'default_drive_group'
+      - item.service_name == 'osd.default_drive_group'
+      - item.data_devices == expected_devices
+  when:
+    - item.service_type == 'osd'
+  loop: "{{ cat_ceph_spec.stdout | from_yaml_all | list }}"
+  vars:
+    expected_hosts:
+      - oc0-ceph-0
+      - oc0-ceph-1
+      - oc0-ceph-2
+    expected_devices:
+      all: true

tripleo_ansible/roles/tripleo_run_cephadm/defaults/main.yml

@@ -1,3 +1,9 @@
 ---
 # defaults file for tripleo_ceph_run_cephadm
 tripleo_run_cephadm_command_log: "cephadm_command.log"
+tripleo_run_cephadm_dynamic_spec: true
+tripleo_run_cephadm_spec_path: "{{ playbook_dir }}/cephadm/ceph_spec.yaml"
+ceph_osd_spec:
+  data_devices:
+    all: true
+ceph_spec_fqdn: false

tripleo_ansible/roles/tripleo_run_cephadm/tasks/prepare.yml

@@ -99,6 +99,15 @@
   vars:
     tripleo_run_cephadm_net: "{{ service_net_map['ceph_mon_network']|default('') + '_ip' }}"
 
+- name: genereate ceph_spec for bootstrap
+  ceph_spec_bootstrap:
+    new_ceph_spec: "{{ tripleo_run_cephadm_spec_path }}"
+    tripleo_ansible_inventory: "{{ inventory_file }}"
+    fqdn: "{{ ceph_spec_fqdn }}"
+    osd_spec: "{{ ceph_osd_spec }}"
+  when:
+    - tripleo_cephadm_dynamic_spec
+
 - name: generate ansible cephadm-extra-vars for running tripleo_cephadm role
   copy:
     dest: "{{ playbook_dir }}/cephadm/cephadm-extra-vars-ansible.yml"
@@ -111,3 +120,5 @@
       tripleo_cephadm_dashboard_frontend_vip: {{ grafana_vip|default() }}
       service_net_map: {{ service_net_map|default({}) }}
       tripleo_enabled_services: {{ enabled_services | default([]) }}
+      tripleo_cephadm_fqdn: "{{ ceph_spec_fqdn | bool }}"
+      tripleo_cephadm_spec_ansible_host: "{{ tripleo_run_cephadm_spec_path }}"

tripleo_ansible/tests/modules/test_ceph_spec_bootstrap.py

@@ -0,0 +1,156 @@
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""Test the methods of the ceph_spec_bootstrap module"""
+
+import yaml
+
+from tripleo_ansible.ansible_plugins.modules import ceph_spec_bootstrap
+from tripleo_ansible.tests import base as tests_base
+
+
+class TestCephSpecBootstrap(tests_base.TestCase):
+    """Test the methods of the ceph_spec_bootstrap module"""
+
+    def test_metal_roles_based_spec(self):
+        """verify we can build a ceph spec and supporting data
+           structures from a mealsmith and tripleo roles file
+        """
+        ceph_service_types = ['mon', 'mgr', 'osd']
+        metal = "roles/tripleo_cephadm/molecule/default/mock/mock_deployed_metal.yaml"
+        tripleo_roles = "roles/tripleo_cephadm/molecule/default/mock/mock_overcloud_roles.yaml"
+        roles_to_svcs = ceph_spec_bootstrap.get_roles_to_svcs_from_roles(tripleo_roles)
+        expected = {
+            'Compute': [],
+            'CephStorage': ['CephOSD'],
+            'Controller': ['CephMgr', 'CephMon']}
+        self.assertEqual(roles_to_svcs, expected)
+
+        roles = roles_to_svcs.keys()
+        roles_to_hosts = ceph_spec_bootstrap.get_deployed_roles_to_hosts(metal, roles)
+        expected = {
+            'Controller': ['oc0-controller-0', 'oc0-controller-1', 'oc0-controller-2'],
+            'Compute': ['oc0-compute-0'],
+            'CephStorage': ['oc0-ceph-0', 'oc0-ceph-1', 'oc0-ceph-2']
+        }
+        self.assertEqual(roles_to_hosts, expected)
+
+        hosts_to_ips = ceph_spec_bootstrap.get_deployed_hosts_to_ips(metal)
+        expected = {'oc0-ceph-0': '192.168.24.13',
+                    'oc0-ceph-1': '192.168.24.11',
+                    'oc0-ceph-2': '192.168.24.14',
+                    'oc0-compute-0': '192.168.24.21',
+                    'oc0-controller-0': '192.168.24.23',
+                    'oc0-controller-1': '192.168.24.15',
+                    'oc0-controller-2': '192.168.24.7'}
+        self.assertEqual(hosts_to_ips, expected)
+
+        label_map = ceph_spec_bootstrap.get_label_map(hosts_to_ips, roles_to_svcs,
+                                                      roles_to_hosts, ceph_service_types)
+        expected = {'oc0-ceph-0': ['osd'],
+                    'oc0-ceph-1': ['osd'],
+                    'oc0-ceph-2': ['osd'],
+                    'oc0-compute-0': [],
+                    'oc0-controller-0': ['mgr', 'mon'],
+                    'oc0-controller-1': ['mgr', 'mon'],
+                    'oc0-controller-2': ['mgr', 'mon']}
+        self.assertEqual(label_map, expected)
+
+        specs = ceph_spec_bootstrap.get_specs(hosts_to_ips, label_map, ceph_service_types)
+        expected = [
+            {'service_type': 'host', 'addr': '192.168.24.13',
+             'hostname': 'oc0-ceph-0', 'labels': ['osd']},
+            {'service_type': 'host', 'addr': '192.168.24.11',
+             'hostname': 'oc0-ceph-1', 'labels': ['osd']},
+            {'service_type': 'host', 'addr': '192.168.24.14',
+             'hostname': 'oc0-ceph-2', 'labels': ['osd']},
+            {'service_type': 'host', 'addr': '192.168.24.23',
+             'hostname': 'oc0-controller-0', 'labels': ['mgr', 'mon']},
+            {'service_type': 'host', 'addr': '192.168.24.15',
+             'hostname': 'oc0-controller-1', 'labels': ['mgr', 'mon']},
+            {'service_type': 'host', 'addr': '192.168.24.7',
+             'hostname': 'oc0-controller-2', 'labels': ['mgr', 'mon']},
+            {
+                'service_type': 'mon',
+                'service_name': 'mon',
+                'service_id': 'mon',
+                'placement': {
+                    'hosts': ['oc0-controller-0', 'oc0-controller-1', 'oc0-controller-2']
+                }
+            },
+            {
+                'service_type': 'mgr',
+                'service_name': 'mgr',
+                'service_id': 'mgr',
+                'placement': {
+                    'hosts': ['oc0-controller-0', 'oc0-controller-1', 'oc0-controller-2']
+                }
+            },
+            {
+                'service_type': 'osd',
+                'service_name': 'osd.default_drive_group',
+                'service_id': 'default_drive_group',
+                'placement': {
+                    'hosts': ['oc0-ceph-0', 'oc0-ceph-1', 'oc0-ceph-2']
+                },
+                'data_devices': {'all': True}
+            }
+        ]
+        self.assertEqual(specs, expected)
+
+    def test_inventory_based_spec(self):
+        """verify we can build a ceph spec and supporting data
+           structures from from a tripleo-ansible inventory
+        """
+        ceph_service_types = ['mon', 'mgr', 'osd']
+        inventory_file = "roles/tripleo_cephadm/molecule/default/mock/mock_inventory.yml"
+        with open(inventory_file, 'r') as stream:
+            inventory = yaml.safe_load(stream)
+        roles_to_svcs = ceph_spec_bootstrap.get_roles_to_svcs_from_inventory(inventory)
+        expected = {'Standalone': ['CephOSD', 'CephMgr', 'CephMon']}
+        self.assertEqual(roles_to_svcs, expected)
+
+        roles = roles_to_svcs.keys()
+        hosts_to_ips = ceph_spec_bootstrap.get_inventory_hosts_to_ips(inventory, roles)
+        expected = {'standalone': '192.168.24.1'}
+        self.assertEqual(hosts_to_ips, expected)
+
+        roles_to_hosts = ceph_spec_bootstrap.get_inventory_roles_to_hosts(inventory, roles)
+        expected = {'Standalone': ['standalone']}
+        self.assertEqual(roles_to_hosts, expected)
+
+        label_map = ceph_spec_bootstrap.get_label_map(hosts_to_ips, roles_to_svcs,
+                                                      roles_to_hosts, ceph_service_types)
+        expected = {'standalone': ['osd', 'mgr', 'mon']}
+        self.assertEqual(label_map, expected)
+
+        specs = ceph_spec_bootstrap.get_specs(hosts_to_ips, label_map, ceph_service_types)
+        expected = [{'addr': '192.168.24.1',
+                     'hostname': 'standalone',
+                     'labels': ['osd', 'mgr', 'mon'],
+                     'service_type': 'host'},
+                    {'placement': {'hosts': ['standalone']},
+                     'service_id': 'mon',
+                     'service_name': 'mon',
+                     'service_type': 'mon'},
+                    {'placement': {'hosts': ['standalone']},
+                     'service_id': 'mgr',
+                     'service_name': 'mgr',
+                     'service_type': 'mgr'},
+                    {'data_devices': {'all': True},
+                     'placement': {'hosts': ['standalone']},
+                     'service_id': 'default_drive_group',
+                     'service_name': 'osd.default_drive_group',
+                     'service_type': 'osd'}]
+        self.assertEqual(specs, expected)