Merge "inventory: Add support for defining custom host groups"

2017-05-09 18:30:41 +00:00 · 2017-05-09 18:30:41 +00:00 · 1ba7641daf
commit 1ba7641daf
parent 15fa11e2af 0233a0e963
5 changed files with 108 additions and 13 deletions
--- a/bifrost/inventory.py
+++ b/bifrost/inventory.py
@ -22,7 +22,7 @@ Bifrost Inventory Module
 This is a dynamic inventory module intended to provide a platform for
 consistent inventory information for Bifrost.

-The inventory currently supplies two distinct groups:
+The inventory supplies two distinct groups by default:

    - localhost
    - baremetal
@ -35,6 +35,21 @@ variables extracted from the data source. The variables are defined on a
 per-host level which allows explicit actions to be taken based upon the
 variables.

+It is also possible for users to specify additional per-host groups by
+simply setting the host_groups variable in the inventory file. See below for
+an example JSON file.
+
+The default group can also be changed by setting the DEFAULT_HOST_GROUPS
+variable to contain the desired groups separated by whitespace as follows:
+
+DEFAULT_HOST_GROUPS="foo bar zoo"
+
+In case of provisioning virtual machines, additional per-VM groups can
+be set by simply setting the test_vm_groups[$host] variable to a list
+of desired groups. Moreover, users can override the default 'baremetal'
+group by assigning a list of default groups to the test_vm_default_group
+variable.
+
 Presently, the base mode of operation reads a CSV file in the format
 originally utilized by bifrost and returns structured JSON that is
 interpreted by Ansible.  This has since been extended to support the
@ -62,6 +77,10 @@ Example JSON Element:
 {
  "node1": {
    "uuid": "a8cb6624-0d9f-c882-affc-046ebb96ec01",
+    "host_groups": [
+        "nova",
+        "neutron"
+    ],
    "driver_info": {
      "power": {
        "ipmi_target_channel": "0",
@ -181,6 +200,11 @@ def _process_baremetal_data(data_source, groups, hostvars):
        # Perform basic validation
        node_net_data = host.get('node_network_data')
        ipv4_addr = host.get('ipv4_address')
+        default_groups = os.environ.get('DEFAULT_HOST_GROUPS',
+                                        'baremetal').split()
+        host['host_groups'] = sorted(list(set(host.get('host_groups', []) +
+                                              default_groups)))
+
        if not node_net_data and not ipv4_addr:
            host['addressing_mode'] = "dhcp"
        else:
@ -190,7 +214,10 @@ def _process_baremetal_data(data_source, groups, hostvars):
                'addressing_mode' not in host):
            host['provisioning_ipv4_address'] = host['ipv4_address']
        # Add each host to the values to be returned.
-        groups['baremetal']['hosts'].append(host['name'])
+        for group in host['host_groups']:
+            if group not in groups:
+                groups.update({group: {'hosts': []}})
+            groups[group]['hosts'].append(host['name'])
        hostvars.update({host['name']: host})
    return (groups, hostvars)

@ -226,6 +253,7 @@ def _process_baremetal_csv(data_source, groups, hostvars):
            properties['cpu_arch'] = "x86_64"
            host['uuid'] = _val_or_none(row, 9)
            host['name'] = _val_or_none(row, 10)
+            host['host_groups'] = ["baremetal"]
            host['ipv4_address'] = _val_or_none(row, 11)
            if ('ipv4_address' not in host or
                    not host['ipv4_address']):
@ -402,6 +430,14 @@ def main():
        LOG.error('Failed processing: %s' % error)
        sys.exit(1)

+    # Drop empty groups. This is usually necessary when
+    # the default ["baremetal"] group has been overridden
+    # by the user.
+    for group in groups.keys():
+        # Empty groups
+        if len(groups[group]['hosts']) == 0:
+            del groups[group]
+
    # General Data Conversion

    if not config.convertcsv:
--- a/bifrost/tests/functional/test_inventory_functional.py
+++ b/bifrost/tests/functional/test_inventory_functional.py
@ -48,7 +48,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
 "ipmi_target_address": null, "ipmi_target_channel": null,
 "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
 [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}, "hostname0":
+ "x86_64", "disk_size": "1024", "cpus": "2"}, "host_groups": ["baremetal"]},
+ "hostname0":
 {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ipmitool",
 "name": "hostname0", "ipv4_address": "192.168.1.2",
 "provisioning_ipv4_address": "192.168.1.2", "ansible_ssh_host":
@ -57,7 +58,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
 "ipmi_target_address": null, "ipmi_target_channel": null,
 "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
 [{"mac": "00:01:02:03:04:05"}], "properties": {"ram": "8192",
- "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"}}}""".replace('\n', '')
+ "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')
        expected_groups = """{"baremetal": {"hosts": ["hostname0",
 "hostname1"]}, "localhost": {"hosts": ["127.0.0.1"]}}""".replace('\n', '')

@ -80,7 +82,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
 "ipmi_transit_address": "40", "ipmi_transit_channel": "30",
 "ipmi_bridging": "dual"}}, "nics":
 [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}}""".replace('\n', '')
+ "x86_64", "disk_size": "1024", "cpus": "2"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')

        expected_groups = """{"baremetal": {"hosts": ["hostname1"]},
 "localhost": {"hosts": ["127.0.0.1"]}}""".replace('\n', '')
@ -104,7 +107,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
 "ipmi_transit_address": null, "ipmi_transit_channel": null,
 "ipmi_bridging": "single"}}, "nics":
 [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}}""".replace('\n', '')
+ "x86_64", "disk_size": "1024", "cpus": "2"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')

        (groups, hostvars) = utils.bifrost_csv_conversion(CSV)
        self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@ -123,7 +127,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1
 "ipmi_target_channel": null, "ipmi_transit_address": null,
 "ipmi_transit_channel": null}}, "nics":
 [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}}""".replace('\n', '')
+ "x86_64", "disk_size": "1024", "cpus": "2"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')

        (groups, hostvars) = utils.bifrost_csv_conversion(CSV)
        self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@ -146,7 +151,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1
 "ipmi_target_channel": null, "ipmi_transit_address": null,
 "ipmi_transit_channel": null}}, "nics":
 [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}}""".replace('\n', '')
+ "x86_64", "disk_size": "1024", "cpus": "2"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')

        (groups, hostvars) = utils.bifrost_csv_conversion(CSV)
        self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@ -170,7 +176,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
 "ipmi_target_address": null, "ipmi_target_channel": null,
 "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
 [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}, "hostname0":
+ "x86_64", "disk_size": "1024", "cpus": "2"}, "host_groups": ["baremetal"]},
+ "hostname0":
 {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ipmitool",
 "name": "hostname0", "ipv4_address": "192.168.1.2", "ansible_ssh_host":
 "192.168.1.2", "provisioning_ipv4_address": "192.168.1.2",
@ -179,7 +186,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
 "ipmi_target_address": null, "ipmi_target_channel": null,
 "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
 [{"mac": "00:01:02:03:04:05"}], "properties": {"ram": "8192",
- "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"}}}""".replace('\n', '')
+ "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"},
+ "host_groups": ["baremetal"]}}""".replace('\n', '')

        (groups, hostvars) = utils.bifrost_csv_conversion(CSV)
        self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@ -200,7 +208,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
 "ipmi_target_address": null, "ipmi_target_channel": null,
 "ipmi_transit_address": null, "ipmi_transit_channel": null}}, "nics":
 [{"mac": "00:01:02:03:04:06"}], "properties": {"ram": "8192", "cpu_arch":
- "x86_64", "disk_size": "1024", "cpus": "2"}}, "hostname0":
+ "x86_64", "disk_size": "1024", "cpus": "2"}, "host_groups":
+ ["baremetal", "nova"]}, "hostname0":
 {"uuid": "00000000-0000-0000-0000-000000000001", "driver": "agent_ssh",
 "name": "hostname0", "ipv4_address": "192.168.1.2", "ansible_ssh_host":
 "192.168.1.2", "provisioning_ipv4_address": "192.168.1.2",
@ -208,7 +217,8 @@ unused,,00000000-0000-0000-0000-000000000002,hostname1,
 "ssh_key_filename": "/home/ironic/.ssh/id_rsa", "ssh_username":
 "ironic", "ssh_port": 22, "ssh_address": "192.0.2.2"}}, "nics":
 [{"mac": "00:01:02:03:04:05"}], "properties": {"ram": "8192",
- "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"}}}""".replace('\n', '')
+ "cpu_arch": "x86_64", "disk_size": "512", "cpus": "1"},
+ "host_groups": ["baremetal", "nova"]}}""".replace('\n', '')
        (groups, hostvars) = utils.bifrost_data_conversion(
            yaml.safe_dump(json.loads(str(expected_hostvars))))
        self.assertDictEqual(json.loads(str(expected_hostvars)), hostvars)
@ -222,6 +232,6 @@ ipmi_password":"ADMIN"}},"driver":"agent_ipmitool"}}""".replace('\n', '')
 "00000000-0000-0000-0001-bad00000010","name":"h0000-01","driver_info"
 :{"power":{"ipmi_address":"10.0.0.78","ipmi_username":"ADMIN","
 ipmi_password":"ADMIN"}},"driver":"agent_ipmitool","addressing_mode":
-"dhcp"}}""".replace('\n', '')
+"dhcp","host_groups": ["baremetal"]}}""".replace('\n', '')
        (groups, hostvars) = utils.bifrost_data_conversion(input_json)
        self.assertDictEqual(json.loads(str(expected_json)), hostvars)
--- a/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml
+++ b/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml
@ -5,6 +5,8 @@ test_vm_memory_size: "3072"
 test_vm_num_nodes: 1
 test_vm_domain_type: "qemu"
 test_vm_arch: "x86_64"
+test_vm_groups: {}
+test_vm_default_groups: "{{ lookup('env', 'DEFAULT_HOST_GROUPS').split() | default(['baremetal'], true) }}"
 test_vm_disk_gib: "{{ lookup('env', 'VM_DISK') | default(10, true) }}"
 test_vm_cpu_count: "{{ lookup('env', 'VM_CPU') | default(1, true) }}"
 test_vm_disk_cache: "{{ lookup('env', 'VM_DISK_CACHE') | default('writeback', true) }}"
--- a/playbooks/roles/bifrost-create-vm-nodes/tasks/create_vm.yml
+++ b/playbooks/roles/bifrost-create-vm-nodes/tasks/create_vm.yml
@ -18,6 +18,11 @@
 - set_fact:
    vm_name: "{{ item }}"
    vm_log_file: "{{ test_vm_logdir }}/{{ item }}_console.log"
+    vm_host_group: "{{ test_vm_default_groups }}"
+
+- set_fact:
+    vm_host_group: "{{ test_vm_default_groups | union(test_vm_groups[vm_name]) }}"
+  when: test_vm_groups[vm_name] is defined

 - name: set prealloc arg for Debian
  set_fact:
@ -126,6 +131,7 @@
    testvm_data:
      name: "{{ vm_name }}"
      uuid: "{{ vm_name | to_uuid }}"
+      host_groups: "{{ vm_host_group }}"
      driver: "{{ test_vm_node_driver|default('agent_ipmitool') }}"
      driver_info:
        power:
--- a/releasenotes/notes/support-custom-ansible-groups-38ad5d3a930b17eb.yaml
+++ b/releasenotes/notes/support-custom-ansible-groups-38ad5d3a930b17eb.yaml
@ -0,0 +1,41 @@
+---
+features:
+  - |
+    It is now possible to define additional per-host inventory
+    groups for all the hosts that make use of the dynamic JSON
+    inventory. The way to do that is to simply define a
+    list of groups in the `host_group` property as illustrated
+    in the following example::
+
+    .. code-block:: yaml
+        "node1": {
+            "uuid": "a8cb6624-0d9f-c882-affc-046ebb96ec01",
+            "host_groups": [
+                "baremetal"
+            ],
+        }
+
+
+    When provisioning virtual machines it's possible to set
+    the per-VM inventory groups by setting the ``test_vm_host_groups``
+    variable as follows::
+
+    ``{ test_vm_host_groups: { testhost: [nova, cinder] } }``
+
+    It is also possible to change the default ``baremetal`` group
+    for virtual machines by simply setting the ``host_default_group``
+    variable to a list of default groups as follows::
+
+    ``{ test_vm_default_groups: [baremetal vms] }``
+
+    The list of default groups can also be set in the
+    ``DEFAULT_HOST_GROUPS`` environmental variable. This is currently
+    the only way to change the default group for baremetal hosts.
+
+    ``export DEFAULT_HOST_GROUPS="foo bar zoo"``
+
+    This will change the default groups to [foo, bar, zoo] instead of
+    the currently [baremetal] default. Extra care should be taken when
+    using this method since most bifrost playbooks depend on having a
+    [baremetal] group available for provisioning hosts.
+