Pass through kolla inventories as is

Previously, we only supported passing through group_vars. Passing
through the inventory as is allows you to use other features of ansible
inventory such as host vars. It also simplifies the logic of merging
multiple inventories as we can just pass the inventory to ansible and
let ansible take care of the rest. This is useful for the multiple
environments feature.

Change-Id: I28f5d73d414d405d67f5fc92ab371aa2e28a4ce3
Story: 2002009
Task: 42910
Depends-On: https://review.opendev.org/c/openstack/kolla-ansible/+/802863

ansible/kolla-ansible.yml

@@ -91,7 +91,9 @@
         kolla_external_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy.pem"
         kolla_internal_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy-internal.pem"
         kolla_ansible_passwords_path: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
-        kolla_overcloud_group_vars_path: "{{ kayobe_env_config_path }}/kolla/inventory/group_vars"
+        kolla_overcloud_inventory_search_paths:
+          - "{{ kayobe_config_path }}"
+          - "{{ kayobe_env_config_path }}"
         kolla_ansible_certificates_path: "{{ kayobe_env_config_path }}/kolla/certificates"
         # NOTE: This differs from the default SELinux mode in kolla ansible,
         # which is permissive. The justification for using this mode is twofold:

ansible/roles/kolla-ansible/defaults/main.yml

@@ -79,8 +79,12 @@ kolla_ansible_become: false
 # Full custom seed inventory contents.
 kolla_seed_inventory_custom:
 
-# Directory containing custom Kolla-Ansible group vars.
-kolla_overcloud_group_vars_path:
+# Directories in kayobe config to search for kolla inventories. The inventory
+# is assumed to be in a directory, 'kolla/inventory', relative to the search path.
+# Any inventories discovered are passed through to kolla-ansible in the order
+# in which they are discovered i.e search paths placed later in the list have
+# precedence over the earlier ones.
+kolla_overcloud_inventory_search_paths: []
 
 # Custom overcloud inventory containing a mapping from top level groups to
 # hosts.

ansible/roles/kolla-ansible/tasks/config.yml

@@ -37,7 +37,7 @@
   with_items:
     - "{{ kolla_config_path }}"
     - "{{ kolla_seed_inventory_path }}"
-    - "{{ kolla_overcloud_inventory_path }}/group_vars"
+    - "{{ kolla_overcloud_inventory_path }}"
     - "{{ kolla_node_custom_config_path }}"
 
 - name: Write environment file into Kolla configuration path
@@ -65,16 +65,52 @@
     dest: "{{ kolla_overcloud_inventory_path }}/hosts"
     mode: 0640
 
-- name: Look for custom Kolla overcloud group vars
-  stat:
-    path: "{{ kolla_overcloud_group_vars_path }}"
-  register: kolla_ansible_custom_overcloud_group_vars
+- name: Make sure extra-inventories directory exists
+  file:
+    path: "{{ kolla_extra_inventories_path }}"
+    mode: "0750"
+    state: directory
 
-- name: Copy over custom Kolla overcloud group vars
-  copy:
-    src: "{{ kolla_overcloud_group_vars_path }}"
-    dest: "{{ kolla_overcloud_inventory_path }}/"
-  when: kolla_ansible_custom_overcloud_group_vars.stat.exists
+- name: Copying custom inventory
+  vars:
+    # This will be the environment name in the case of a kayobe environment
+    inventory_name: "{{ (item  ~ '/../..') | realpath | basename }}"
+  synchronize:
+    dest: "{{ kolla_extra_inventories_path }}/{{ inventory_name }}"
+    recursive: true
+    delete: true
+    src: "{{ item }}/"
+    rsync_opts:
+      - --exclude=kayobe_blank_hosts
+      - --exclude=*.j2
+  loop: "{{ kolla_overcloud_inventory_search_paths | product(['/kolla/inventory']) | map('join') | select('exists') | unique | list }}"
+  loop_control:
+    label: "{{ inventory_name }}"
+
+- name: Create blank hosts file to prevent ansible warning
+  # Silence a benign warning: Unable to parse
+  # <kolla-config-path>/extra-inventories/level2/inventory as an inventory source
+  # When no hosts are defined. This occurs when you only define group_vars.
+  vars:
+    inventory_name: "{{ (item  ~ '/../..') | realpath | basename }}"
+  file:
+    path: "{{ kolla_extra_inventories_path }}/{{ inventory_name }}/kayobe_blank_hosts"
+    state: touch
+    modification_time: preserve
+    access_time: preserve
+  loop: "{{ kolla_overcloud_inventory_search_paths | product(['/kolla/inventory']) | map('join') | select('exists') | unique | list }}"
+  loop_control:
+    label: "{{ inventory_name }}"
+
+- name: Clean up inventories that no longer exist
+  vars:
+    inventory_name: "{{ (item  ~ '/../..') | realpath | basename }}"
+  file:
+    path: "{{ kolla_extra_inventories_path }}/{{ inventory_name }}"
+    state: absent
+  loop: "{{ kolla_overcloud_inventory_search_paths | product(['/kolla/inventory']) | map('join') | reject('exists') | unique | list }}"
+  loop_control:
+    label: "{{ inventory_name }}"
 
 - name: Ensure the Kolla passwords file exists
   vars:

ansible/roles/kolla-ansible/tests/test-defaults.yml

@@ -22,7 +22,8 @@
             kolla_node_custom_config_path: "{{ temp_path }}/etc/kolla/config"
             # Purposely does not exist to simulate the case when no group vars
             # are provided
-            kolla_overcloud_group_vars_path: "{{ temp_path }}/etc/kayobe/kolla/inventory/group_vars"
+            kolla_overcloud_inventory_search_paths:
+              - "{{ temp_path }}/etc/kayobe/"
             kolla_ansible_passwords_path: "{{ temp_path }}/passwords.yml"
             # Required config.
             kolla_base_distro: "fake-distro"
@@ -128,27 +129,17 @@
           with_items:
             - seed
             - overcloud
-            - overcloud/group_vars
           register: inventory_stat
 
-        - name: Validate inventory files
-          assert:
-            that:
-              - item.stat.exists
-              - item.stat.size > 0
-            msg: >
-              Inventory file {{ item.item }} was not found.
-          with_items: "{{ inventory_stat.results }}"
-
-        - name: Look for custom overcloud group vars
+        - name: Look for inventory overrides
           find:
-            paths: "{{ temp_path ~ '/etc/kolla/inventory/group_vars' }}"
-          register: kolla_ansible_overcloud_group_vars
+            paths: "{{ temp_path ~ '/etc/kolla/extra-inventories/' }}"
+          register: kolla_ansible_overcloud_inventory_overrides
 
-        - name: Check that no overcloud group vars are set
+        - name: Check that no inventory overrides are configured
           assert:
             that:
-              - kolla_ansible_overcloud_group_vars.matched == 0
+              -  kolla_ansible_overcloud_inventory_overrides.matched == 0
             msg: >
               Overcloud group vars were found when they should not be set.
 

ansible/roles/kolla-ansible/tests/test-extras.yml

@@ -50,6 +50,19 @@
           ---
           bar_port: "4567"
 
+    - name: Create directory for extra group vars
+      file:
+        path: "{{ tempfile_result.path ~ '/etc/kayobe/environments/example/kolla/inventory/group_vars' }}"
+        recurse: true
+        state: directory
+
+    - name: Create custom extra group vars
+      copy:
+        dest: "{{ tempfile_result.path ~ '/etc/kayobe/environments/example/kolla/inventory/group_vars/baz_group' }}"
+        content: |
+          ---
+          baz_port: "8910"
+
     - name: Create directory for custom CA certificates
       file:
         path: "{{ tempfile_result.path }}/etc/kayobe/kolla/certificates/ca"
@@ -79,7 +92,9 @@
             kolla_ansible_venv: "{{ temp_path }}/venv"
             kolla_ansible_vault_password: "fake-password"
             kolla_config_path: "{{ temp_path }}/etc/kolla"
-            kolla_overcloud_group_vars_path: "{{ temp_path }}/etc/kayobe/kolla/inventory/group_vars"
+            kolla_overcloud_inventory_search_paths:
+              - "{{ temp_path }}/etc/kayobe/"
+              - "{{ temp_path }}/etc/kayobe/environments/example/"
             kolla_node_custom_config_path: "{{ temp_path }}/etc/kolla/config"
             kolla_ansible_passwords_path: "{{ temp_path }}/passwords.yml"
             # Config.
@@ -417,14 +432,21 @@
             - test-controller
             - test-compute
 
-        - name: Check whether inventory group vars files exist
+        - name: Check whether inventory group vars from base config exist
           stat:
-            path: "{{ temp_path ~ '/etc/kolla/inventory/overcloud/group_vars/' ~ item }}"
+            path: "{{ temp_path ~ '/etc/kolla/extra-inventories/kayobe/group_vars/' ~ item }}"
           with_items:
             - foo_group/all
             - bar_group
           register: group_vars_stat
 
+        - name: Check whether inventory group vars from environment exist
+          stat:
+            path: "{{ temp_path ~ '/etc/kolla/extra-inventories/example/group_vars/' ~ item }}"
+          with_items:
+            - baz_group
+          register: group_vars_environment_stat
+
         - name: Validate inventory group vars files
           assert:
             that:
@@ -432,7 +454,7 @@
               - item.stat.size > 0
             msg: >
               Inventory file {{ item.item }} was not found.
-          with_items: "{{ group_vars_stat.results }}"
+          with_items: "{{ group_vars_stat.results + group_vars_environment_stat.results }}"
 
         - name: Read inventory group vars files
           slurp:
@@ -440,6 +462,12 @@
           with_items: "{{ group_vars_stat.results }}"
           register: group_vars_slurp
 
+        - name: Read inventory environment group vars files
+          slurp:
+            src: "{{ item.stat.path }}"
+          with_items: "{{ group_vars_environment_stat.results }}"
+          register: group_vars_environment_slurp
+
         - name: Validate inventory group vars file contents
           assert:
             that:
@@ -458,6 +486,21 @@
                 ---
                 bar_port: "4567"
 
+        - name: Validate environment inventory group vars file contents
+          assert:
+            that:
+              - group_vars_content is defined
+              - group_vars_content == item.1
+          with_together:
+            - "{{ group_vars_environment_slurp.results }}"
+            - "{{ expected_contents }}"
+          vars:
+            group_vars_content: "{{ item.0.content | b64decode }}"
+            expected_contents:
+              - |
+                ---
+                baz_port: "8910"
+
         - name: Check whether API certificate files exist
           stat:
             path: "{{ temp_path ~ '/etc/kolla/certificates/' ~ item }}"

ansible/roles/kolla-ansible/vars/Debian.yml

@@ -7,3 +7,4 @@ kolla_ansible_package_dependencies:
   - python3-dev
   - python3-pip
   - python3-venv
+  - rsync

ansible/roles/kolla-ansible/vars/RedHat.yml

@@ -6,3 +6,4 @@ kolla_ansible_package_dependencies:
   - openssl-devel
   - python3-devel
   - python3-pip
+  - rsync

ansible/roles/kolla-ansible/vars/main.yml

@@ -66,6 +66,10 @@ kolla_seed_inventory_path: "{{ kolla_config_path }}/inventory/seed"
 # Path to the kolla ansible overcloud inventory directory.
 kolla_overcloud_inventory_path: "{{ kolla_config_path }}/inventory/overcloud"
 
+# Path to pass-through inventories. These are layered on top of kayobe
+# generated one.
+kolla_extra_inventories_path: "{{ kolla_config_path }}/extra-inventories"
+
 ###############################################################################
 # Feature configuration.
 

doc/source/configuration/reference/kolla-ansible.rst

@@ -540,9 +540,26 @@ In case the variable requires a different name in Kolla Ansible, use
    kolla_overcloud_inventory_pass_through_host_vars_map_extra:
      my_kayobe_var: my_kolla_ansible_var
 
-Custom Group Variables
+.. _custom_kolla_inventory:
+
+Custom Kolla Inventory
 ----------------------
 
+When running Kolla Ansible playbooks, kayobe will check for any customised
+inventories in the following locations:
+
+* ``${KAYOBE_CONFIG_PATH}/kolla/inventory/``
+* ``${KAYOBE_CONFIG_PATH}/environments/<environment>/kolla/inventory/``
+    * Only used with the :ref:`multiple environments feature <multiple-environments>`
+
+These are copied when kayobe generates the Kolla Ansible configuration. The
+copy is passed to Ansible as an additional inventory when running any
+Kolla Ansible playbooks. No templating or additional preprocessing is
+performed. For this reason, this directory must be a valid Ansible inventory,
+with the exception that ``*.j2`` files are ignored to keep compatibility with
+:ref:`custom Kolla Ansible inventory templates
+<custom-kolla-inventory-templates>`.
+
 Group variables can be used to set configuration for all hosts in a group. They
 can be set in Kolla Ansible by placing files in
 ``${KAYOBE_CONFIG_PATH}/kolla/inventory/group_vars/*``. Since this

doc/source/control-plane-service-placement.rst

@@ -210,6 +210,8 @@ providing the necessary variables for a control plane host.
 Here we are using the controller-specific values for some of these variables,
 but they could equally be different.
 
+.. _custom-kolla-inventory-templates:
+
 Example 2: Overriding the Kolla-ansible Inventory
 -------------------------------------------------
 

doc/source/multiple-environments.rst

@@ -1,3 +1,5 @@
+.. _multiple-environments:
+
 =====================
 Multiple Environments
 =====================
@@ -64,6 +66,12 @@ Kayobe configuration.
            ├── networks.yml
            └── overcloud.yml
 
+Naming
+------
+
+The environment name ``kayobe`` is reserved for internal use. The name should
+be a valid directory name, otherwise there are no other restrictions.
+
 Ansible Inventories
 -------------------
 
@@ -91,6 +99,22 @@ files) shows an example of multiple inventories.
        ├── groups
        └── group_vars/
 
+Custom Kolla Ansible inventories
+--------------------------------
+
+Kayobe has a :ref:`feature <custom_kolla_inventory>` to pass through
+additional inventories to Kolla Ansible. When using multiple environments,
+these are passed though as additional inventories to Ansible. The ordering is
+such that the inventory in the base layer of kayobe config overrides the
+internal kayobe inventory, and inventory in the environment overrides inventory
+in the base layer:
+
+.. code-block:: bash
+
+   ansible-playbook -i <internal kayobe inventory> -i <inventory from base layer> -i <inventory from environment>
+
+See :ref:`custom_kolla_inventory` for more details.
+
 Shared Extra Variables Files
 ----------------------------
 

kayobe/ansible.py

@@ -124,6 +124,11 @@ def _validate_args(parsed_args, playbooks):
                   parsed_args.config_path, result["message"])
         sys.exit(1)
 
+    if parsed_args.environment and parsed_args.environment == "kayobe":
+        LOG.error("The environment name 'kayobe' is reserved for internal "
+                  "use.")
+        sys.exit(1)
+
     env_path = _get_kayobe_environment_path(parsed_args)
     if env_path:
         result = utils.is_readable_dir(env_path)

kayobe/kolla_ansible.py

@@ -52,7 +52,8 @@ def add_args(parser):
                         help="specify inventory host path "
                              "(default=$%s/inventory or %s/inventory) for "
                              "Kolla Ansible" %
-                             (CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH))
+                             (CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH),
+                        action='append')
     parser.add_argument("-kl", "--kolla-limit", metavar="SUBSET",
                         help="further limit selected hosts to an additional "
                              "pattern")
@@ -70,13 +71,30 @@ def add_args(parser):
                              (VENV_PATH_ENV, DEFAULT_VENV_PATH))
 
 
-def _get_inventory_path(parsed_args, inventory_filename):
+def _get_inventory_paths(parsed_args, inventory_filename):
     """Return the path to the Kolla inventory."""
     if parsed_args.kolla_inventory:
         return parsed_args.kolla_inventory
     else:
-        return os.path.join(parsed_args.kolla_config_path, "inventory",
-                            inventory_filename)
+        paths = [os.path.join(parsed_args.kolla_config_path, "inventory",
+                              inventory_filename)]
+
+        def append_path(directory):
+            candidate_path = os.path.join(
+                parsed_args.kolla_config_path, "extra-inventories",
+                directory)
+            if utils.is_readable_dir(candidate_path)["result"]:
+                paths.append(candidate_path)
+
+        # Inventory in the base layer is placed in the "kayobe"
+        # directory. This means that you can't have an environment
+        # called kayobe as it would conflict.
+        append_path("kayobe")
+
+        if parsed_args.environment:
+            append_path(parsed_args.environment)
+
+        return paths
 
 
 def _validate_args(parsed_args, inventory_filename):
@@ -88,17 +106,18 @@ def _validate_args(parsed_args, inventory_filename):
                   parsed_args.kolla_config_path, result["message"])
         sys.exit(1)
 
-    inventory = _get_inventory_path(parsed_args, inventory_filename)
-    result = utils.is_readable_dir(inventory)
-    if not result["result"]:
-        # NOTE(mgoddard): Previously the inventory was a file, now it is a
-        # directory to allow us to support inventory host_vars. Support both
-        # formats for now.
-        result_f = utils.is_readable_file(inventory)
-        if not result_f["result"]:
-            LOG.error("Kolla inventory %s is invalid: %s",
-                      inventory, result["message"])
-            sys.exit(1)
+    inventories = _get_inventory_paths(parsed_args, inventory_filename)
+    for inventory in inventories:
+        result = utils.is_readable_dir(inventory)
+        if not result["result"]:
+            # NOTE(mgoddard): Previously the inventory was a file, now it is a
+            # directory to allow us to support inventory host_vars. Support
+            # both formats for now.
+            result_f = utils.is_readable_file(inventory)
+            if not result_f["result"]:
+                LOG.error("Kolla inventory %s is invalid: %s",
+                          inventory, result["message"])
+                sys.exit(1)
 
     result = utils.is_readable_dir(parsed_args.kolla_venv)
     if not result["result"]:
@@ -125,8 +144,9 @@ def build_args(parsed_args, command, inventory_filename, extra_vars=None,
     if parsed_args.kolla_playbook:
         cmd += ["--playbook", parsed_args.kolla_playbook]
     cmd += vault.build_args(parsed_args, "--key")
-    inventory = _get_inventory_path(parsed_args, inventory_filename)
-    cmd += ["--inventory", inventory]
+    inventories = _get_inventory_paths(parsed_args, inventory_filename)
+    for inventory in inventories:
+        cmd += ["--inventory", inventory]
     if parsed_args.kolla_config_path != DEFAULT_CONFIG_PATH:
         cmd += ["--configdir", parsed_args.kolla_config_path]
         cmd += ["--passwords",

kayobe/tests/unit/test_ansible.py

@@ -14,6 +14,7 @@
 
 import argparse
 import errno
+import logging
 import os
 import os.path
 import shutil
@@ -92,6 +93,53 @@ class TestCase(unittest.TestCase):
                                          quiet=False, env=expected_env)
         mock_vars.assert_called_once_with(["/etc/kayobe"])
 
+    @mock.patch.object(ansible, "_get_vars_files")
+    @mock.patch.object(utils, "is_readable_dir")
+    @mock.patch.object(utils, "is_readable_file")
+    @mock.patch.object(utils, "run_command")
+    def test_reserved_environment(
+            self, mock_run, mock_readable,
+            mock_readable_file, mock_vars):
+        mock_readable_file.return_value = {"result": True}
+        mock_readable.return_value = {"result": True}
+        mock_vars.return_value = ["/path/to/config/vars-file1.yml",
+                                  "/path/to/config/vars-file2.yaml"]
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        vault.add_args(parser)
+        args = [
+            "--environment", "kayobe",
+        ]
+        parsed_args = parser.parse_args(args)
+        with self.assertLogs(level=logging.ERROR) as ctx:
+            self.assertRaises(
+                SystemExit, ansible.run_playbooks, parsed_args,
+                ["playbook1.yml"]
+            )
+            exp = "The environment name 'kayobe' is reserved for internal use."
+            log_found = any(exp in t for t in ctx.output)
+            assert(log_found)
+
+    @mock.patch.object(ansible, "_get_vars_files")
+    @mock.patch.object(utils, "is_readable_dir")
+    @mock.patch.object(utils, "is_readable_file")
+    @mock.patch.object(utils, "run_command")
+    def test_reserved_environment_negative(
+            self, mock_run, mock_readable,
+            mock_readable_file, mock_vars):
+        mock_readable_file.return_value = {"result": True}
+        mock_readable.return_value = {"result": True}
+        mock_vars.return_value = ["/path/to/config/vars-file1.yml",
+                                  "/path/to/config/vars-file2.yaml"]
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        vault.add_args(parser)
+        args = [
+            "--environment", "kayobe2",
+        ]
+        parsed_args = parser.parse_args(args)
+        ansible.run_playbooks(parsed_args, ["playbook1.yml", "playbook2.yml"])
+
     @mock.patch.object(utils, "run_command")
     @mock.patch.object(ansible, "_get_vars_files")
     @mock.patch.object(ansible, "_validate_args")

kayobe/tests/unit/test_kolla_ansible.py

@@ -240,6 +240,40 @@ class TestCase(unittest.TestCase):
                                          env=expected_env)
         mock_readable.assert_called_once_with("/etc/kayobe/kolla/ansible.cfg")
 
+    @mock.patch.object(utils, "run_command")
+    @mock.patch.object(utils, "is_readable_dir")
+    @mock.patch.object(utils, "is_readable_file")
+    @mock.patch.object(kolla_ansible, "_validate_args")
+    def test_run_environment_inventories(self, mock_validate, mock_readable,
+                                         mock_readable_dir, mock_run):
+        mock_readable.return_value = {"result": True}
+        mock_readable_dir.return_value = {"result": True}
+        parser = argparse.ArgumentParser()
+        ansible.add_args(parser)
+        kolla_ansible.add_args(parser)
+        vault.add_args(parser)
+        args = [
+            "--environment", "myenv",
+        ]
+        parsed_args = parser.parse_args(args)
+        kolla_ansible.run(parsed_args, "command", "overcloud")
+        expected_cmd = [
+            ".", "/path/to/cwd/venvs/kolla-ansible/bin/activate", "&&",
+            "kolla-ansible", "command",
+            "--inventory", "/etc/kolla/inventory/overcloud",
+            "--inventory", "/etc/kolla/extra-inventories/kayobe",
+            "--inventory", '/etc/kolla/extra-inventories/myenv'
+        ]
+        expected_cmd = " ".join(expected_cmd)
+        mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False,
+                                         env=mock.ANY)
+        mock_readable_dir.assert_any_call(
+            "/etc/kolla/extra-inventories/kayobe"
+        )
+        mock_readable_dir.assert_any_call(
+            "/etc/kolla/extra-inventories/myenv"
+        )
+
     @mock.patch.object(utils, "run_command")
     @mock.patch.object(utils, "is_readable_file")
     @mock.patch.object(kolla_ansible, "_validate_args")

releasenotes/notes/adds-support-for-multiple-kolla-inventories-65fd7a4922c639c5.yaml

@@ -0,0 +1,21 @@
+---
+features:
+  - |
+    Kolla Ansible inventories in the Kayobe configuration are now passed
+    through without modification.  Previously, only ``group_vars`` were passed
+    through.  When using multiple environments, the Kolla inventory from the
+    base configuration layer **and** the Kolla inventory from the Kayobe
+    environment layer will be passed through. The inventory from the
+    environment takes precedence over the inventory from the base layer. This
+    allows you to put any shared configuration in the base layer.
+upgrade:
+  - |
+    As Kolla Ansible inventories are now passed through without modification,
+    the inventory directory in Kayobe configuration
+    (``etc/kayobe/kolla/inventory/``) must be a valid Ansible inventory,
+    although ``*.j2`` files used as Kolla Ansible inventory templates are
+    ignored. For cases where only ``group_vars`` or ``hosts_vars`` are
+    required, a blank inventory file in the same directory may be used.
+  - |
+    It is no longer possible to create an environment named ``kayobe``. This
+    is reserved for internal use.