module tripleo_baremetal_expand_roles

This module consumes the baremetal deployment yaml and transforms it
to a list of instances and a heat environment. Subsequent modules in
the role will consume the instances and take actions based on the
instance entries and the observed ironic nodes.

The mistral action[1] has significant unit test coverage[2] which is
desirable to keep, so most of the logic has been put in a module_utils
with its own unit tests. Some special handling is required in the unit
test and doc generation to make baremetal_deploy importable from
ansible.module_utils, which is done automatically during ansible runs.

[1] https://opendev.org/openstack/tripleo-common/src/branch/master/tripleo_common/actions/baremetal_deploy.py#L353
[2] https://opendev.org/openstack/tripleo-common/src/branch/master/tripleo_common/tests/actions/test_baremetal_deploy.py#L648

Change-Id: I21cc6939db5120d4c1549b9ec66d6e0f172fd229
Story: 2007212
Task: 38457

ansible-requirements.txt

@@ -1 +1,3 @@
 ansible>=2.8
+metalsmith>=0.13.0 # Apache-2.0
+jsonschema  # MIT

doc/source/conf.py

@@ -15,6 +15,8 @@
 import os
 import sys
 
+from ansible.plugins import loader
+
 # Add the project
 sys.path.insert(0, os.path.abspath('../..'))
 # Add the extensions
@@ -84,3 +86,14 @@ latex_documents = [
 repository_name = 'openstack/tripleo-ansible'
 bug_project = 'tripleo'
 bug_tag = 'documentation'
+
+needed_module_utils = [
+    'baremetal_deploy'
+]
+# load our custom module_utils so that modules can be imported for
+# generating docs
+for m in needed_module_utils:
+    try:
+        loader.module_utils_loader.get(m)
+    except AttributeError:
+        pass

doc/source/modules/modules-tripleo_baremetal_expand_roles.rst

@@ -0,0 +1,14 @@
+=======================================
+Module - tripleo_baremetal_expand_roles
+=======================================
+
+
+This module provides for the following ansible plugin:
+
+    * tripleo_baremetal_expand_roles
+
+
+.. ansibleautoplugin::
+   :module: tripleo_ansible/ansible_plugins/modules/tripleo_baremetal_expand_roles.py
+   :documentation: true
+   :examples: true

molecule-requirements.txt

@@ -21,3 +21,5 @@ openstackdocstheme>=1.29.2 # Apache-2.0
 reno>=2.11.3 # Apache-2.0
 doc8>=0.8.0 # Apache-2.0
 bashate>=0.6.0 # Apache-2.0
+metalsmith>=0.13.0 # Apache-2.0
+jsonschema  # MIT

tripleo_ansible/ansible_plugins/module_utils/baremetal_deploy.py

@@ -0,0 +1,300 @@
+#!/usr/bin/python
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import jsonschema
+
+from metalsmith import sources
+
+
+_IMAGE_SCHEMA = {
+    'type': 'object',
+    'properties': {
+        'href': {'type': 'string'},
+        'checksum': {'type': 'string'},
+        'kernel': {'type': 'string'},
+        'ramdisk': {'type': 'string'},
+    },
+    'required': ['href'],
+    'additionalProperties': False,
+}
+
+_NIC_SCHEMA = {
+    'type': 'object',
+    'properties': {
+        'network': {'type': 'string'},
+        'port': {'type': 'string'},
+        'fixed_ip': {'type': 'string'},
+        'subnet': {'type': 'string'},
+    },
+    'additionalProperties': False
+}
+
+_INSTANCE_SCHEMA = {
+    'type': 'object',
+    'properties': {
+        'capabilities': {'type': 'object'},
+        'conductor_group': {'type': 'string'},
+        'hostname': {
+            'type': 'string',
+            'minLength': 2,
+            'maxLength': 255
+        },
+        'image': _IMAGE_SCHEMA,
+        'name': {'type': 'string'},
+        'netboot': {'type': 'boolean'},
+        'nics': {
+            'type': 'array',
+            'items': _NIC_SCHEMA
+        },
+        'passwordless_sudo': {'type': 'boolean'},
+        'profile': {'type': 'string'},
+        'provisioned': {'type': 'boolean'},
+        'resource_class': {'type': 'string'},
+        'root_size_gb': {'type': 'integer', 'minimum': 4},
+        'ssh_public_keys': {'type': 'string'},
+        'swap_size_mb': {'type': 'integer', 'minimum': 64},
+        'traits': {
+            'type': 'array',
+            'items': {'type': 'string'}
+        },
+        'user_name': {'type': 'string'},
+    },
+    'additionalProperties': False,
+}
+
+
+_INSTANCES_SCHEMA = {
+    'type': 'array',
+    'items': _INSTANCE_SCHEMA
+}
+"""JSON schema of the instances list."""
+
+
+_ROLES_INPUT_SCHEMA = {
+    'type': 'array',
+    'items': {
+        'type': 'object',
+        'properties': {
+            'name': {'type': 'string'},
+            'hostname_format': {'type': 'string'},
+            'count': {'type': 'integer', 'minimum': 0},
+            'defaults': _INSTANCE_SCHEMA,
+            'instances': _INSTANCES_SCHEMA,
+        },
+        'additionalProperties': False,
+        'required': ['name'],
+    }
+}
+"""JSON schema of the roles list."""
+
+
+class BaremetalDeployException(Exception):
+    pass
+
+
+def expand(roles, stack_name, expand_provisioned=True, default_image=None,
+           default_network=None, user_name=None, ssh_public_keys=None):
+
+    for role in roles:
+        role.setdefault('defaults', {})
+        if default_image:
+            role['defaults'].setdefault('image', default_image)
+        if default_network:
+            role['defaults'].setdefault('nics', default_network)
+        for inst in role.get('instances', []):
+            for k, v in role['defaults'].items():
+                inst.setdefault(k, v)
+
+            # Set the default hostname now for duplicate hostname
+            # detection during validation
+            if 'hostname' not in inst and 'name' in inst:
+                inst['hostname'] = inst['name']
+
+    validate_roles(roles)
+
+    instances = []
+    hostname_map = {}
+    parameter_defaults = {'HostnameMap': hostname_map}
+    for role in roles:
+        name = role['name']
+        hostname_format = build_hostname_format(
+            role.get('hostname_format'), name)
+        count = role.get('count', 1)
+        unprovisioned_indexes = []
+
+        # build a map of all potential generated names
+        # with the index number which generates the name
+        potential_gen_names = {}
+        for index in range(count + len(role.get('instances', []))):
+            potential_gen_names[build_hostname(
+                hostname_format, index, stack_name)] = index
+
+        # build a list of instances from the specified
+        # instances list
+        role_instances = []
+        for instance in role.get('instances', []):
+            inst = {}
+            inst.update(instance)
+
+            # create a hostname map entry now if the specified hostname
+            # is a valid generated name
+            if inst.get('hostname') in potential_gen_names:
+                hostname_map[inst['hostname']] = inst['hostname']
+
+            if ssh_public_keys:
+                inst['ssh_public_keys'] = ssh_public_keys
+            if user_name:
+                inst['user_name'] = user_name
+
+            role_instances.append(inst)
+
+        # add generated instance entries until the desired count of
+        # provisioned instances is reached
+        while len([i for i in role_instances
+                   if i.get('provisioned', True)]) < count:
+            inst = {}
+            inst.update(role['defaults'])
+            role_instances.append(inst)
+
+        # NOTE(dtantsur): our hostname format may differ from THT defaults,
+        # so override it in the resulting environment
+        parameter_defaults['%sDeployedServerHostnameFormat' % name] = (
+            hostname_format)
+
+        # ensure each instance has a unique non-empty hostname
+        # and a hostname map entry. Also build a list of indexes
+        # for unprovisioned instances
+        index = 0
+        for inst in role_instances:
+            provisioned = inst.get('provisioned', True)
+            gen_name = None
+            hostname = inst.get('hostname')
+
+            if hostname not in hostname_map:
+                while (not gen_name
+                        or gen_name in hostname_map):
+                    gen_name = build_hostname(
+                        hostname_format, index, stack_name)
+                    index += 1
+                inst.setdefault('hostname', gen_name)
+                hostname = inst.get('hostname')
+                hostname_map[gen_name] = inst['hostname']
+
+            if not provisioned:
+                if gen_name:
+                    unprovisioned_indexes.append(
+                        potential_gen_names[gen_name])
+                elif hostname in potential_gen_names:
+                    unprovisioned_indexes.append(
+                        potential_gen_names[hostname])
+
+        if unprovisioned_indexes:
+            parameter_defaults['%sRemovalPolicies' % name] = [{
+                'resource_list': unprovisioned_indexes
+            }]
+
+        provisioned_count = 0
+        for inst in role_instances:
+            provisioned = inst.pop('provisioned', True)
+
+            if provisioned:
+                provisioned_count += 1
+
+            # Only add instances which match the desired provisioned state
+            if provisioned == expand_provisioned:
+                instances.append(inst)
+
+        parameter_defaults['%sDeployedServerCount' % name] = (
+            provisioned_count)
+
+    validate_instances(instances)
+    if expand_provisioned:
+        env = {'parameter_defaults': parameter_defaults}
+    else:
+        env = {}
+    return instances, env
+
+
+def build_hostname_format(hostname_format, role_name):
+    if not hostname_format:
+        hostname_format = '%stackname%-{}-%index%'.format(
+            'novacompute' if role_name == 'Compute' else role_name.lower())
+    return hostname_format
+
+
+def build_hostname(hostname_format, index, stack):
+    gen_name = hostname_format.replace('%index%', str(index))
+    gen_name = gen_name.replace('%stackname%', stack)
+    return gen_name
+
+
+def validate_instances(instances):
+    jsonschema.validate(instances, _INSTANCES_SCHEMA)
+    hostnames = set()
+    names = set()
+    for inst in instances:
+        # NOTE(dtantsur): validate image parameters
+        get_source(inst)
+
+        if inst.get('hostname'):
+            if inst['hostname'] in hostnames:
+                raise ValueError('Hostname %s is used more than once' %
+                                 inst['hostname'])
+            hostnames.add(inst['hostname'])
+
+        if inst.get('name'):
+            if inst['name'] in names:
+                raise ValueError('Node %s is requested more than once' %
+                                 inst['name'])
+            names.add(inst['name'])
+
+
+def validate_roles(roles):
+    jsonschema.validate(roles, _ROLES_INPUT_SCHEMA)
+
+    for item in roles:
+        count = item.get('count', 1)
+        instances = item.get('instances', [])
+        instances = [i for i in instances if i.get('provisioned', True)]
+        name = item.get('name')
+        if len(instances) > count:
+            raise ValueError(
+                "%s: number of instance entries %s "
+                "cannot be greater than count %s" %
+                (name, len(instances), count)
+            )
+
+        defaults = item.get('defaults', {})
+        if 'hostname' in defaults:
+            raise ValueError("%s: cannot specify hostname in defaults"
+                             % name)
+        if 'name' in defaults:
+            raise ValueError("%s: cannot specify name in defaults"
+                             % name)
+        if 'provisioned' in defaults:
+            raise ValueError("%s: cannot specify provisioned in defaults"
+                             % name)
+        if 'instances' in item:
+            validate_instances(item['instances'])
+
+
+def get_source(instance):
+    image = instance.get('image', {})
+    return sources.detect(image=image.get('href'),
+                          kernel=image.get('kernel'),
+                          ramdisk=image.get('ramdisk'),
+                          checksum=image.get('checksum'))

tripleo_ansible/ansible_plugins/modules/tripleo_baremetal_expand_roles.py

@@ -0,0 +1,253 @@
+#!/usr/bin/python
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+__metaclass__ = type
+
+from ansible.module_utils import baremetal_deploy as bd
+from ansible.module_utils.basic import AnsibleModule
+
+import yaml
+
+
+ANSIBLE_METADATA = {
+    'metadata_version': '1.1',
+    'status': ['preview'],
+    'supported_by': 'community'
+}
+
+
+DOCUMENTATION = '''
+---
+module: tripleo_baremetal_expand_roles
+short_description: Manage baremetal nodes with metalsmith
+version_added: "2.9"
+author: "Steve Baker (@stevebaker)"
+description:
+  - Takes a baremetal deployment description of roles and node instances
+    and transforms that into an instance list and a heat environment file
+    for deployed-server.
+options:
+  stack_name:
+    description:
+      - Name of the overcloud stack which will be deployed on these instances
+    default: overcloud
+  state:
+    description:
+      - Build instance list for the desired provision state, "present" to
+        provision, "absent" to unprovision, "all" for a combination of
+        "present" and "absent".
+    default: present
+    choices:
+    - present
+    - absent
+    - all
+  baremetal_deployment:
+    description:
+      - Data describing roles and baremetal node instances to provision for
+        those roles
+    type: list
+    elements: dict
+    suboptions:
+      name:
+        description:
+          - Mandatory role name
+        type: str
+        required: True
+      hostname_format:
+        description:
+          - Overrides the default hostname format for this role.
+            The default format uses the lower case role name.
+            For example, the default format for the Controller role is
+            %stackname%-controller-%index%. Only the Compute role does not
+            follow the role name rule. The Compute default format is
+            %stackname%-novacompute-%index%
+        type: str
+      count:
+        description:
+          - Number of instances to create for this role.
+        type: int
+        default: 1
+      defaults:
+        description:
+          - A dictionary of default values for instances entry properties.
+            An instances entry property overrides any defaults that you specify
+            in the defaults parameter.
+        type: dict
+      instances:
+        description:
+          - Values that you can use to specify attributes for specific nodes.
+            The length of this list must not be greater than the value of the
+            count parameter.
+        type: list
+        elements: dict
+  default_network:
+    description:
+      - Default nics entry when none are specified
+    type: list
+    suboptions: dict
+    default:
+      - network: ctlplane
+  default_image:
+    description:
+      - Default image
+    type: dict
+    default:
+      href: overcloud-full
+  ssh_public_keys:
+    description:
+      - SSH public keys to load
+    type: str
+  user_name:
+    description:
+      - Name of the admin user to create
+    type: str
+'''
+
+RETURN = '''
+instances:
+    description: Expanded list of instances to perform actions on
+    returned: changed
+    type: list
+    sample: [
+            {
+                "hostname": "overcloud-controller-0",
+                "image": {
+                    "href": "overcloud-full"
+                }
+            },
+            {
+                "hostname": "overcloud-controller-1",
+                "image": {
+                    "href": "overcloud-full"
+                }
+            },
+            {
+                "hostname": "overcloud-controller-2",
+                "image": {
+                    "href": "overcloud-full"
+                }
+            },
+            {
+                "hostname": "overcloud-novacompute-0",
+                "image": {
+                    "href": "overcloud-full"
+                }
+            },
+            {
+                "hostname": "overcloud-novacompute-1",
+                "image": {
+                    "href": "overcloud-full"
+                }
+            },
+            {
+                "hostname": "overcloud-novacompute-2",
+                "image": {
+                    "href": "overcloud-full"
+                }
+            }
+        ]
+environment:
+    description: Heat environment data to be used with the overcloud deploy.
+                 This is only a partial environment, further changes are
+                 required once instance changes have been made.
+    returned: changed
+    type: dict
+    sample: {
+            "parameter_defaults": {
+                "ComputeDeployedServerCount": 3,
+                "ComputeDeployedServerHostnameFormat": "%stackname%-novacompute-%index%",
+                "ControllerDeployedServerCount": 3,
+                "ControllerDeployedServerHostnameFormat": "%stackname%-controller-%index%",
+                "HostnameMap": {
+                    "overcloud-controller-0": "overcloud-controller-0",
+                    "overcloud-controller-1": "overcloud-controller-1",
+                    "overcloud-controller-2": "overcloud-controller-2",
+                    "overcloud-novacompute-0": "overcloud-novacompute-0",
+                    "overcloud-novacompute-1": "overcloud-novacompute-1",
+                    "overcloud-novacompute-2": "overcloud-novacompute-2"
+                }
+            }
+        }
+'''  # noqa
+
+EXAMPLES = '''
+- name: Expand roles
+  tripleo_baremetal_expand_roles:
+    baremetal_deployment:
+    - name: Controller
+      count: 3
+      defaults:
+        image:
+          href: overcloud-full
+    - name: Compute
+      count: 3
+      defaults:
+        image:
+          href: overcloud-full
+    state: present
+    stack_name: overcloud
+  register: tripleo_baremetal_instances
+'''
+
+
+def main():
+    argument_spec = yaml.safe_load(DOCUMENTATION)['options']
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=False,
+    )
+
+    state = module.params['state']
+
+    try:
+        if state in ('present', 'all'):
+            present, env = bd.expand(
+                roles=module.params['baremetal_deployment'],
+                stack_name=module.params['stack_name'],
+                expand_provisioned=True,
+                default_image=module.params['default_image'],
+                default_network=module.params['default_network'],
+                user_name=module.params['user_name'],
+                ssh_public_keys=module.params['ssh_public_keys'],
+            )
+        if state in ('absent', 'all'):
+            absent, _ = bd.expand(
+                roles=module.params['baremetal_deployment'],
+                stack_name=module.params['stack_name'],
+                expand_provisioned=False,
+                default_image=module.params['default_image'],
+            )
+            env = {}
+        if state == 'present':
+            instances = present
+        elif state == 'absent':
+            instances = absent
+        elif state == 'all':
+            instances = present + absent
+
+        module.exit_json(
+            changed=True,
+            msg='Expanded to %d instances' % len(instances),
+            instances=instances,
+            environment=env
+        )
+    except Exception as e:
+        module.fail_json(msg=str(e))
+
+
+if __name__ == '__main__':
+    main()

tripleo_ansible/tests/base.py

@@ -13,9 +13,23 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from ansible.plugins import loader
+
 from oslotest import base
 
 
-class TestCase(base.BaseTestCase):
+def load_module_utils(*args):
+    """Ensure requested module_utils are loaded into ansible.module_utils"""
+    if args:
+        for m in args:
+            try:
+                loader.module_utils_loader.get(m)
+            except AttributeError:
+                pass
+    else:
+        # search and load all module_utils, its noisy and slower
+        list(loader.module_utils_loader.all())
 
+
+class TestCase(base.BaseTestCase):
     """Test case base class for all unit tests."""

tripleo_ansible/tests/plugins/module_utils/test_baremetal_deploy.py

@@ -0,0 +1,672 @@
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from tripleo_ansible.tests import base
+
+# load baremetal_deploy so the next import works
+base.load_module_utils('baremetal_deploy')
+
+from ansible.module_utils import baremetal_deploy as bd  # noqa
+
+
+class TestBaremetalDeployUtils(base.TestCase):
+
+    def test_build_hostname_format(self):
+        self.assertEqual(
+            '%stackname%-controller-%index%',
+            bd.build_hostname_format(None, 'Controller')
+        )
+        self.assertEqual(
+            '%stackname%-novacompute-%index%',
+            bd.build_hostname_format(None, 'Compute')
+        )
+        self.assertEqual(
+            'server-%index%',
+            bd.build_hostname_format('server-%index%', 'Compute')
+        )
+
+    def test_build_hostname(self):
+        self.assertEqual(
+            'overcloud-controller-2',
+            bd.build_hostname(
+                '%stackname%-controller-%index%', 2, 'overcloud'
+            )
+        )
+        self.assertEqual(
+            'server-2',
+            bd.build_hostname(
+                'server-%index%', 2, 'overcloud'
+            )
+        )
+
+
+class TestExpandRoles(base.TestCase):
+
+    default_image = {'href': 'overcloud-full'}
+
+    def test_simple(self):
+        roles = [
+            {'name': 'Compute'},
+            {'name': 'Controller'},
+        ]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+
+        self.assertEqual(
+            [
+                {'hostname': 'overcloud-novacompute-0',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'overcloud-controller-0',
+                 'image': {'href': 'overcloud-full'}},
+            ],
+            instances)
+        self.assertEqual(
+            {
+                'ComputeDeployedServerHostnameFormat':
+                '%stackname%-novacompute-%index%',
+                'ComputeDeployedServerCount': 1,
+                'ControllerDeployedServerHostnameFormat':
+                '%stackname%-controller-%index%',
+                'ControllerDeployedServerCount': 1,
+                'HostnameMap': {
+                    'overcloud-novacompute-0': 'overcloud-novacompute-0',
+                    'overcloud-controller-0': 'overcloud-controller-0'
+                }
+            },
+            environment['parameter_defaults'])
+
+    def test_image_in_defaults(self):
+        roles = [{
+            'name': 'Controller',
+            'defaults': {
+                'image': {
+                    'href': 'file:///tmp/foo.qcow2',
+                    'checksum': '12345678'
+                }
+            },
+            'count': 3,
+            'instances': [{
+                'hostname': 'overcloud-controller-0',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'overcloud-controller-1',
+            }]
+        }]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual(
+            [
+                {'hostname': 'overcloud-controller-0',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'overcloud-controller-1',
+                 'image': {'href': 'file:///tmp/foo.qcow2',
+                           'checksum': '12345678'}},
+                {'hostname': 'overcloud-controller-2',
+                 'image': {'href': 'file:///tmp/foo.qcow2',
+                           'checksum': '12345678'}},
+            ],
+            instances)
+
+    def test_with_parameters(self):
+        roles = [{
+            'name': 'Compute',
+            'count': 2,
+            'defaults': {
+                'profile': 'compute'
+            },
+            'hostname_format': 'compute-%index%.example.com'
+        }, {
+            'name': 'Controller',
+            'count': 3,
+            'defaults': {
+                'profile': 'control'
+            },
+            'hostname_format': 'controller-%index%.example.com'
+        }]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual(
+            [
+                {'hostname': 'compute-0.example.com', 'profile': 'compute',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'compute-1.example.com', 'profile': 'compute',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'controller-0.example.com', 'profile': 'control',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'controller-1.example.com', 'profile': 'control',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'controller-2.example.com', 'profile': 'control',
+                 'image': {'href': 'overcloud-full'}},
+            ],
+            instances)
+        self.assertEqual(
+            {
+                'ComputeDeployedServerHostnameFormat':
+                'compute-%index%.example.com',
+                'ComputeDeployedServerCount': 2,
+                'ControllerDeployedServerHostnameFormat':
+                'controller-%index%.example.com',
+                'ControllerDeployedServerCount': 3,
+                'HostnameMap': {
+                    'compute-0.example.com': 'compute-0.example.com',
+                    'compute-1.example.com': 'compute-1.example.com',
+                    'controller-0.example.com': 'controller-0.example.com',
+                    'controller-1.example.com': 'controller-1.example.com',
+                    'controller-2.example.com': 'controller-2.example.com',
+                }
+            },
+            environment['parameter_defaults'])
+
+    def test_explicit_instances(self):
+        roles = [{
+            'name': 'Compute',
+            'count': 2,
+            'defaults': {
+                'profile': 'compute'
+            },
+            'hostname_format': 'compute-%index%.example.com'
+        }, {
+            'name': 'Controller',
+            'count': 2,
+            'defaults': {
+                'profile': 'control'
+            },
+            'instances': [{
+                'hostname': 'controller-X.example.com',
+                'profile': 'control-X'
+            }, {
+                'name': 'node-0',
+                'traits': ['CUSTOM_FOO'],
+                'nics': [{'subnet': 'leaf-2'}]},
+            ]},
+        ]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual(
+            [
+                {'hostname': 'compute-0.example.com', 'profile': 'compute',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'compute-1.example.com', 'profile': 'compute',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'controller-X.example.com',
+                 'image': {'href': 'overcloud-full'},
+                 'profile': 'control-X'},
+                # Name provides the default for hostname later on.
+                {'name': 'node-0', 'profile': 'control',
+                 'hostname': 'node-0',
+                 'image': {'href': 'overcloud-full'},
+                 'traits': ['CUSTOM_FOO'], 'nics': [{'subnet': 'leaf-2'}]},
+            ],
+            instances)
+        self.assertEqual(
+            {
+                'ComputeDeployedServerHostnameFormat':
+                'compute-%index%.example.com',
+                'ComputeDeployedServerCount': 2,
+                'ControllerDeployedServerHostnameFormat':
+                '%stackname%-controller-%index%',
+                'ControllerDeployedServerCount': 2,
+                'HostnameMap': {
+                    'compute-0.example.com': 'compute-0.example.com',
+                    'compute-1.example.com': 'compute-1.example.com',
+                    'overcloud-controller-0': 'controller-X.example.com',
+                    'overcloud-controller-1': 'node-0',
+                }
+            },
+            environment['parameter_defaults'])
+
+    def test_count_with_instances(self):
+        roles = [{
+            'name': 'Compute',
+            'count': 2,
+            'defaults': {
+                'profile': 'compute',
+            },
+            'hostname_format': 'compute-%index%.example.com'
+        }, {
+            'name': 'Controller',
+            'defaults': {
+                'profile': 'control',
+            },
+            'count': 3,
+            'instances': [{
+                'hostname': 'controller-X.example.com',
+                'profile': 'control-X'
+            }, {
+                'name': 'node-0',
+                'traits': ['CUSTOM_FOO'],
+                'nics': [{'subnet': 'leaf-2'}]},
+            ]},
+        ]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'compute-0.example.com',
+                'profile': 'compute',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'compute-1.example.com',
+                'profile': 'compute',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'controller-X.example.com',
+                'profile': 'control-X',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'node-0',
+                'name': 'node-0',
+                'nics': [{'subnet': 'leaf-2'}],
+                'profile': 'control',
+                'traits': ['CUSTOM_FOO'],
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'overcloud-controller-2',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({
+            'ComputeDeployedServerCount': 2,
+            'ComputeDeployedServerHostnameFormat':
+                'compute-%index%.example.com',
+            'ControllerDeployedServerCount': 3,
+            'ControllerDeployedServerHostnameFormat':
+                '%stackname%-controller-%index%',
+            'HostnameMap': {
+                'compute-0.example.com': 'compute-0.example.com',
+                'compute-1.example.com': 'compute-1.example.com',
+                'overcloud-controller-0': 'controller-X.example.com',
+                'overcloud-controller-1': 'node-0',
+                'overcloud-controller-2': 'overcloud-controller-2'}
+            },
+            environment['parameter_defaults'])
+
+    def test_unprovisioned(self):
+        roles = [{
+            'name': 'Controller',
+            'defaults': {
+                'profile': 'control',
+            },
+            'count': 2,
+            'instances': [{
+                'hostname': 'overcloud-controller-1',
+                'provisioned': False
+            }, {
+                'hostname': 'overcloud-controller-2',
+                'provisioned': False
+            }]
+        }]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'overcloud-controller-0',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'overcloud-controller-3',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({
+            'ControllerDeployedServerCount': 2,
+            'ControllerRemovalPolicies': [
+                {'resource_list': [1, 2]}
+            ],
+            'ControllerDeployedServerHostnameFormat':
+                '%stackname%-controller-%index%',
+            'HostnameMap': {
+                'overcloud-controller-0': 'overcloud-controller-0',
+                'overcloud-controller-1': 'overcloud-controller-1',
+                'overcloud-controller-2': 'overcloud-controller-2',
+                'overcloud-controller-3': 'overcloud-controller-3'}
+            },
+            environment['parameter_defaults'])
+
+        instances, environment = bd.expand(
+            roles, 'overcloud', False, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'overcloud-controller-1',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'overcloud-controller-2',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({}, environment)
+
+    def test_reprovisioned(self):
+        roles = [{
+            'name': 'Controller',
+            'defaults': {
+                'profile': 'control',
+            },
+            'count': 4,
+            'instances': [{
+                'hostname': 'overcloud-controller-1',
+                'provisioned': False
+            }, {
+                'hostname': 'overcloud-controller-2',
+                'provisioned': False
+            }]
+        }]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'overcloud-controller-0',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'overcloud-controller-3',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'overcloud-controller-4',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'overcloud-controller-5',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({
+            'ControllerDeployedServerCount': 4,
+            'ControllerRemovalPolicies': [
+                {'resource_list': [1, 2]}
+            ],
+            'ControllerDeployedServerHostnameFormat':
+                '%stackname%-controller-%index%',
+            'HostnameMap': {
+                'overcloud-controller-0': 'overcloud-controller-0',
+                'overcloud-controller-1': 'overcloud-controller-1',
+                'overcloud-controller-2': 'overcloud-controller-2',
+                'overcloud-controller-3': 'overcloud-controller-3',
+                'overcloud-controller-4': 'overcloud-controller-4',
+                'overcloud-controller-5': 'overcloud-controller-5'}
+            },
+            environment['parameter_defaults'])
+
+        instances, environment = bd.expand(
+            roles, 'overcloud', False, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'overcloud-controller-1',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'overcloud-controller-2',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({}, environment)
+
+    def test_unprovisioned_instances(self):
+        roles = [{
+            'name': 'Controller',
+            'defaults': {
+                'profile': 'control',
+            },
+            'count': 2,
+            'instances': [{
+                'name': 'node-0',
+                'hostname': 'controller-0'
+            }, {
+                'name': 'node-1',
+                'hostname': 'controller-1',
+                'provisioned': False
+            }, {
+                'name': 'node-2',
+                'hostname': 'controller-2',
+                'provisioned': False
+            }, {
+                'name': 'node-3',
+                'hostname': 'controller-3',
+                'provisioned': True
+            }]
+        }]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'controller-0',
+                'name': 'node-0',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'controller-3',
+                'name': 'node-3',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({
+            'ControllerDeployedServerCount': 2,
+            'ControllerRemovalPolicies': [
+                {'resource_list': [1, 2]}
+            ],
+            'ControllerDeployedServerHostnameFormat':
+                '%stackname%-controller-%index%',
+            'HostnameMap': {
+                'overcloud-controller-0': 'controller-0',
+                'overcloud-controller-1': 'controller-1',
+                'overcloud-controller-2': 'controller-2',
+                'overcloud-controller-3': 'controller-3'}
+            },
+            environment['parameter_defaults'])
+
+        instances, environment = bd.expand(
+            roles, 'overcloud', False, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'controller-1',
+                'name': 'node-1',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'controller-2',
+                'name': 'node-2',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({}, environment)
+
+    def test_unprovisioned_no_hostname(self):
+        roles = [{
+            'name': 'Controller',
+            'defaults': {
+                'profile': 'control',
+            },
+            'count': 2,
+            'instances': [{
+                'name': 'node-0',
+            }, {
+                'name': 'node-1',
+                'provisioned': False
+            }, {
+                'name': 'node-2',
+                'provisioned': False
+            }, {
+                'name': 'node-3',
+                'provisioned': True
+            }]
+        }]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'node-0',
+                'name': 'node-0',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'node-3',
+                'name': 'node-3',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({
+            'ControllerDeployedServerCount': 2,
+            'ControllerRemovalPolicies': [
+                {'resource_list': [1, 2]}
+            ],
+            'ControllerDeployedServerHostnameFormat':
+                '%stackname%-controller-%index%',
+            'HostnameMap': {
+                'overcloud-controller-0': 'node-0',
+                'overcloud-controller-1': 'node-1',
+                'overcloud-controller-2': 'node-2',
+                'overcloud-controller-3': 'node-3'}
+            },
+            environment['parameter_defaults'])
+
+        instances, environment = bd.expand(
+            roles, 'overcloud', False, self.default_image
+        )
+        self.assertEqual([
+            {
+                'hostname': 'node-1',
+                'name': 'node-1',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }, {
+                'hostname': 'node-2',
+                'name': 'node-2',
+                'profile': 'control',
+                'image': {'href': 'overcloud-full'}
+            }],
+            instances)
+        self.assertEqual({}, environment)
+
+    def test_name_in_defaults(self):
+        roles = [{
+            'name': 'Compute',
+            'count': 2,
+            'defaults': {
+                'profile': 'compute',
+                'name': 'compute-0'
+            }
+        }]
+        exc = self.assertRaises(
+            ValueError, bd.expand,
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertIn('Compute: cannot specify name in defaults',
+                      str(exc))
+
+    def test_hostname_in_defaults(self):
+        roles = [{
+            'name': 'Compute',
+            'count': 2,
+            'defaults': {
+                'profile': 'compute',
+                'hostname': 'compute-0'
+            }
+        }]
+        exc = self.assertRaises(
+            ValueError, bd.expand,
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertIn('Compute: cannot specify hostname in defaults',
+                      str(exc))
+
+    def test_instances_without_hostname(self):
+        roles = [{
+            'name': 'Compute',
+            'count': 2,
+            'defaults': {
+                'profile': 'compute'
+            },
+            'hostname_format': 'compute-%index%.example.com'
+        }, {
+            'name': 'Controller',
+            'count': 2,
+            'defaults': {
+                'profile': 'control'
+            },
+            'instances': [{
+                'profile': 'control-X'
+                # missing hostname here
+            }, {
+                'name': 'node-0',
+                'traits': ['CUSTOM_FOO'],
+                'nics': [{'subnet': 'leaf-2'}]},
+            ]},
+        ]
+        instances, environment = bd.expand(
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertEqual(
+            [
+                {'hostname': 'compute-0.example.com', 'profile': 'compute',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'compute-1.example.com', 'profile': 'compute',
+                 'image': {'href': 'overcloud-full'}},
+                {'hostname': 'overcloud-controller-0', 'profile': 'control-X',
+                 'image': {'href': 'overcloud-full'}},
+                # Name provides the default for hostname
+                {'name': 'node-0', 'profile': 'control',
+                 'hostname': 'node-0',
+                 'image': {'href': 'overcloud-full'},
+                 'traits': ['CUSTOM_FOO'], 'nics': [{'subnet': 'leaf-2'}]},
+            ],
+            instances)
+
+    def test_more_instances_than_count(self):
+        roles = [{
+            'name': 'Compute',
+            'count': 3,
+            'defaults': {
+                'profile': 'compute',
+                'name': 'compute-0'
+            },
+            'instances': [{
+                'name': 'node-0'
+            }, {
+                'name': 'node-1'
+            }, {
+                'name': 'node-2'
+            }, {
+                'name': 'node-3'
+            }]
+        }]
+        exc = self.assertRaises(
+            ValueError, bd.expand,
+            roles, 'overcloud', True, self.default_image
+        )
+        self.assertIn('Compute: number of instance entries 4 '
+                      'cannot be greater than count 3',
+                      str(exc))