Use merge_configs and merge_yaml to generate Kolla configs

Supports merging configuration for the following files:

* kolla/globals.yml
* kolla/config/bifrost/bifrost.yml
* kolla/config/bifrost/dib.yml
* kolla/config/bifrost/servers.yml
* kolla/kolla-build.conf

Configuration is merged from the following sources:

* Kayobe source code
* Base Kayobe config
* Kayobe environment

Co-Authored-By: Will Szumski <will@stackhpc.com>
Change-Id: I552bd8f7853b2032954b372bf4476676dac3e271
Story: 2002009
Task: 42974

ansible/kolla-ansible.yml

@@ -51,19 +51,6 @@
             - kolla_environment_file.stat.exists
             - kolla_environment != kayobe_environment
 
-        # Configuration of extra user-provided Kolla globals.
-        - name: Check whether a Kolla extra globals configuration file exists
-          stat:
-            path: "{{ kayobe_env_config_path ~ '/kolla/globals.yml' }}"
-            get_checksum: False
-            get_md5: False
-            mime: False
-          register: globals_stat
-
-        - name: Read the Kolla extra globals configuration file
-          set_fact:
-            kolla_extra_globals: "{{ lookup('template', kayobe_env_config_path ~ '/kolla/globals.yml') | from_yaml }}"
-          when: globals_stat.stat.exists
       tags:
         - config
 
@@ -117,6 +104,9 @@
         kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}"
         kolla_enable_host_ntp: false
         docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}"
+        kolla_globals_paths_extra:
+          - "{{ kayobe_config_path }}"
+          - "{{ kayobe_env_config_path }}"
 
 - name: Generate Kolla Ansible host vars for the seed host
   hosts: seed

ansible/kolla-bifrost.yml

@@ -3,23 +3,6 @@
   hosts: localhost
   tags:
     - kolla-bifrost
-  vars:
-    kolla_bifrost_extra_globals_path: "{{ kayobe_env_config_path ~ '/kolla/config/bifrost/bifrost.yml' }}"
-
-  pre_tasks:
-    - name: Check whether a Kolla Bifrost extra globals configuration file exists
-      stat:
-        path: "{{ kolla_bifrost_extra_globals_path }}"
-        get_checksum: False
-        get_md5: False
-        mime: False
-      register: globals_stat
-
-    - name: Read the Kolla Bifrost extra globals configuration file
-      set_fact:
-        kolla_bifrost_extra_globals: "{{ lookup('template', kolla_bifrost_extra_globals_path) | from_yaml }}"
-      when: globals_stat.stat.exists
-
   roles:
     - role: kolla-bifrost
 
@@ -30,3 +13,6 @@
       kolla_bifrost_dnsmasq_dns_servers: "{{ resolv_nameservers | default([]) }}"
       kolla_bifrost_domain: "{{ resolv_domain | default }}"
       kolla_bifrost_download_ipa: "{{ not ipa_build_images | bool }}"
+      kolla_bifrost_config_paths_extra:
+        - "{{ kayobe_config_path }}"
+        - "{{ kayobe_env_config_path }}"

ansible/kolla-build.yml

@@ -7,4 +7,6 @@
     - role: kolla
       kolla_install_epel: "{{ dnf_install_epel }}"
     - role: kolla-build
-      kolla_build_extra_config_path: "{{ kayobe_env_config_path }}/kolla/kolla-build.conf"
+      kolla_build_config_paths_extra:
+        - "{{ kayobe_config_path }}"
+        - "{{ kayobe_env_config_path }}"

ansible/roles/kolla-ansible/defaults/main.yml

@@ -231,9 +231,25 @@ kolla_nova_compute_ironic_host:
 ###############################################################################
 # Extra free-form configuraton.
 
+# Deprecated:
 # Free form extra configuration to append to {{ kolla_config_path }}/globals.yml.
 kolla_extra_globals:
 
+# List of paths to YAML files containing extra configuration to merge to {{
+# kolla_config_path }}/globals.yml. Default is include the globals.yml template
+# from the role.
+kolla_globals_paths_default:
+  - "{{ role_path }}/templates"
+
+# List of paths to YAML files containing extra configuration to merge to {{
+# kolla_config_path }}/globals.yml. Default is an empty list.
+kolla_globals_paths_extra: []
+
+# List of paths to YAML files containing extra configuration to merge to {{
+# kolla_config_path }}/globals.yml. Default is combination of
+# kolla_globals_paths_default and kolla_globals_paths_extra.
+kolla_globals_paths: "{{ kolla_globals_paths_default + kolla_globals_paths_extra }}"
+
 # Dictionary containing custom passwords to add or override in the Kolla
 # passwords file.
 kolla_ansible_custom_passwords: {}

ansible/roles/kolla-ansible/tasks/config.yml

@@ -48,8 +48,8 @@
   when: (kayobe_environment | default('')) | length > 0
 
 - name: Ensure the Kolla global configuration file exists
-  template:
-    src: "globals.yml.j2"
+  merge_yaml:
+    sources: "{{ kolla_globals_paths | product(['/kolla/globals.yml']) | map('join') | unique | list }}"
     dest: "{{ kolla_config_path }}/globals.yml"
     mode: 0640
   vars:

ansible/roles/kolla-ansible/templates/kolla/globals.yml

@@ -1,6 +1,4 @@
 ---
-# {{ ansible_managed }}
-
 # You can use this file to override _any_ variable throughout Kolla.
 # Additional options can be found in the
 # 'kolla-ansible/ansible/group_vars/all.yml' file. Default value of all the

ansible/roles/kolla-ansible/tests/main.yml

@@ -9,6 +9,7 @@
 - import_playbook: test-defaults.yml
 - import_playbook: test-extras.yml
 - import_playbook: test-requirements.yml
+- import_playbook: test-globals-merge.yml
 
 - hosts: localhost
   connection: local

ansible/roles/kolla-ansible/tests/test-defaults.yml

@@ -38,6 +38,8 @@
             kolla_enable_tls_internal: False
             kolla_enable_grafana: False
             kolla_openstack_logging_debug: False
+            kolla_globals_paths_extra:
+              - "{{ tempfile_result.path ~ '/etc/kayobe/' }}"
             apt_cache_valid_time: 3600
 
         - name: Verify kolla-ansible installation

ansible/roles/kolla-ansible/tests/test-extras.yml

@@ -28,6 +28,14 @@
         path: "{{ tempfile_result.path ~ '/etc/kayobe/kolla/inventory/group_vars/foo_group' }}"
         state: directory
 
+    - name: Create extra globals file
+      copy:
+        content: |
+          ---
+          extra-global-1: "extra-val-1"
+          extra-global-2: "extra-val-2"
+        dest: "{{ tempfile_result.path ~ '/etc/kayobe/kolla/globals.yml' }}"
+
     - name: Create custom overcloud foo group vars
       copy:
         dest: "{{ tempfile_result.path ~ '/etc/kayobe/kolla/inventory/group_vars/foo_group/all' }}"
@@ -181,9 +189,8 @@
             kolla_enable_watcher: True
             kolla_enable_zookeeper: True
             kolla_enable_zun: True
-            kolla_extra_globals:
-              extra-global-1: "extra-val-1"
-              extra-global-2: "extra-val-2"
+            kolla_globals_paths_extra:
+              - "{{ tempfile_result.path ~ '/etc/kayobe/' }}"
             kolla_ansible_custom_passwords:
               custom-password-1: "custom-password-1"
               custom-password-2: "custom-password-2"

ansible/roles/kolla-ansible/tests/test-globals-merge.yml

@@ -0,0 +1,116 @@
+---
+- name: Test kolla-ansible role defaults
+  hosts: localhost
+  connection: local
+  tasks:
+    - name: Create a temporary directory
+      tempfile:
+        state: directory
+      register: tempfile_result
+
+    - block:
+
+        - name: Ensure directories exists
+          file:
+            state: directory
+            mode: "0700"
+            recurse: true
+            path: "{{ item }}"
+          with_items:
+            - "{{ tempfile_result.path }}/etc/kayobe/kolla/"
+            - "{{ tempfile_result.path }}//etc/kayobe/environments/level1/kolla"
+            - "{{ tempfile_result.path }}//etc/kayobe/environments/level2/kolla"
+
+        - name: Write contents to base globals.yml
+          copy:
+            content: |
+              _overridden_level_1_var: base
+              _base_var: base
+            dest: "{{ tempfile_result.path }}/etc/kayobe/kolla/globals.yml"
+
+        - name: Write contents to level1 globals.yml
+          copy:
+            content: |
+              _overridden_level_1_var: level1
+              _overridden_level_2_var: level1
+            dest: "{{ tempfile_result.path }}//etc/kayobe/environments/level1/kolla/globals.yml"
+
+        - name: Write contents to level2 globals.yml
+          copy:
+            content: |
+              _overridden_level_2_var: level2
+            dest: "{{ tempfile_result.path }}//etc/kayobe/environments/level2/kolla/globals.yml"
+
+        - name: Test the kolla-ansible role with default values
+          include_role:
+            name: ../../kolla-ansible
+          vars:
+            kolla_ansible_source_path: "{{ temp_path }}/src"
+            kolla_ansible_ctl_install_type: "source"
+            kolla_ansible_source_url: "http://github.com/openstack/kolla-ansible"
+            kolla_ansible_source_version: "{{ openstack_branch }}"
+            kolla_ansible_venv: "{{ temp_path }}/venv"
+            kolla_config_path: "{{ temp_path }}/etc/kolla"
+            kolla_node_custom_config_path: "{{ temp_path }}/etc/kolla/config"
+            # Purposely does not exist to simulate the case when no group vars
+            # are provided
+            kolla_overcloud_group_vars_path: "{{ temp_path }}/etc/kayobe/kolla/inventory/group_vars"
+            kolla_ansible_passwords_path: "{{ temp_path }}/passwords.yml"
+            # Required config.
+            kolla_base_distro: "fake-distro"
+            kolla_install_type: "fake-install-type"
+            kolla_docker_namespace: "fake-namespace"
+            kolla_openstack_release: "fake-release"
+            kolla_internal_vip_address: "10.0.0.1"
+            kolla_internal_fqdn: "fake.internal.fqdn"
+            kolla_external_vip_address: "10.0.0.2"
+            kolla_external_fqdn: "fake.external.fqdn"
+            kolla_ansible_certificates_path: "{{ temp_path }}/etc/kayobe/kolla/certificates"
+            kolla_enable_tls_external: False
+            kolla_enable_tls_internal: False
+            kolla_enable_grafana: False
+            kolla_openstack_logging_debug: False
+            kolla_globals_paths_extra:
+              - "{{ tempfile_result.path ~ '/etc/kayobe/' }}"
+              - "{{ tempfile_result.path ~ '/etc/kayobe/environments/level1/' }}"
+              - "{{ tempfile_result.path ~ '/etc/kayobe/environments/level2/' }}"
+            apt_cache_valid_time: 3600
+
+        - name: Verify kolla-ansible installation
+          shell: ". {{ temp_path }}/venv/bin/activate && kolla-ansible -h"
+          changed_when: False
+
+        - name: Verify ansible installation
+          command: "{{ temp_path }}/venv/bin/ansible -h"
+          changed_when: False
+
+        - name: Validate globals.yml contents
+          assert:
+            that:
+              - item.key in globals_yml
+              - globals_yml[item.key] == item.value
+            msg: >
+              Unexpected value for variable "{{ item.key }}" in globals.yml.
+              Expected "{{ item.value }}", actual
+              "{{ globals_yml.get(item.key, '<missing>') }}".
+          with_dict: "{{ expected_variables }}"
+          vars:
+            # NOTE: Can't use set_fact for this, as it causes kolla-ansible
+            # Jinja expressions to be evaluated.
+            globals_yml: "{{ lookup('file', temp_path ~ '/etc/kolla/globals.yml') | from_yaml }}"
+            expected_variables:
+              _overridden_level_1_var: level1
+              _overridden_level_2_var: level2
+              _base_var: base
+
+      always:
+        - name: Ensure the temporary directory is removed
+          file:
+            path: "{{ temp_path }}"
+            state: absent
+      rescue:
+        - name: Flag that a failure occurred
+          set_fact:
+            test_failures: "{{ test_failures | default(0) | int + 1 }}"
+      vars:
+        temp_path: "{{ tempfile_result.path }}"

ansible/roles/kolla-ansible/tests/test-requirements.yml

@@ -37,6 +37,8 @@
             kolla_enable_tls_internal: False
             kolla_enable_grafana: False
             kolla_openstack_logging_debug: False
+            kolla_globals_paths_extra:
+              - "{{ tempfile_result.path ~ '/etc/kayobe/' }}"
             apt_cache_valid_time: 3600
 
         - name: List Python packages installed in virtualenv

ansible/roles/kolla-bifrost/defaults/main.yml

@@ -74,5 +74,17 @@ kolla_bifrost_ipa_ramdisk_checksum_algorithm:
 # Server inventory to be configured in {{ kolla_node_custom_config_path }}/bifrost/servers.yml.
 kolla_bifrost_servers: {}
 
+# Deprecated.
 # Free form extra configuration to append to {{ kolla_node_custom_config_path }}/bifrost/bifrost.yml.
 kolla_bifrost_extra_globals:
+
+# Paths to Kolla Ansible custom configuration.
+kolla_bifrost_config_paths_default:
+  - "{{ role_path }}/templates"
+
+# Paths to Kolla Ansible custom configuration.
+kolla_bifrost_config_paths_extra: []
+
+# Paths to Kolla Ansible custom configuration. Defaults to a combination of
+# kolla_bifrost_config_paths_default and kolla_bifrost_config_paths_extra.
+kolla_bifrost_config_paths: "{{ kolla_bifrost_config_paths_default + kolla_bifrost_config_paths_extra }}"

ansible/roles/kolla-bifrost/tasks/main.yml

@@ -6,11 +6,11 @@
     mode: 0750
 
 - name: Ensure the Kolla Bifrost configuration files exist
-  template:
-    src: "{{ item.src }}"
-    dest: "{{ kolla_node_custom_config_path }}/bifrost/{{ item.dest }}"
+  merge_yaml:
+    sources: "{{ kolla_bifrost_config_paths | product(['/kolla/config/bifrost/' ~ item]) | map('join') | list }}"
+    dest: "{{ kolla_node_custom_config_path }}/bifrost/{{ item }}"
     mode: 0640
   with_items:
-    - { src: bifrost.yml.j2, dest: bifrost.yml }
-    - { src: dib.yml.j2, dest: dib.yml }
-    - { src: servers.yml.j2, dest: servers.yml }
+    - bifrost.yml
+    - dib.yml
+    - servers.yml

ansible/roles/kolla-build/defaults/main.yml

@@ -2,8 +2,15 @@
 # Directory where Kolla config files will be installed.
 kolla_build_config_path:
 
-# Path to extra kolla configuration files.
-kolla_build_extra_config_path:
+# Paths to extra kolla configuration files.
+kolla_build_config_paths_default:
+  - "{{ role_path }}/templates/"
+
+# Paths to extra kolla configuration files.
+kolla_build_config_paths_extra: []
+
+# Paths to extra kolla configuration files.
+kolla_build_config_paths: "{{ kolla_build_config_paths_default + kolla_build_config_paths_extra }}"
 
 # Valid options are [ centos, fedora, oraclelinux, ubuntu ]
 kolla_base_distro:

ansible/roles/kolla-build/tasks/main.yml

@@ -1,23 +1,12 @@
 ---
-- name: Check whether a Kolla build extra configuration file exists
-  local_action:
-    module: stat
-    path: "{{ kolla_build_extra_config_path }}"
-    get_checksum: False
-    get_md5: False
-    mime: False
-  register: stat_result
-
-- name: Set a fact containing extra configuration
-  set_fact:
-    kolla_build_extra_config: "{{ lookup('template', kolla_build_extra_config_path) }}"
-  when: stat_result.stat.exists
-
-- name: Ensure the Kolla build configuration files exist
-  template:
-    src: "{{ item.src }}"
-    dest: "{{ kolla_build_config_path }}/{{ item.dest }}"
+- name: Ensure the Kolla build configuration file exists
+  merge_configs:
+    sources: "{{ kolla_build_config_paths | product(['/kolla/kolla-build.conf']) | map('join') | list }}"
+    dest: "{{ kolla_build_config_path }}/kolla-build.conf"
+    mode: 0644
+
+- name: Ensure the Kolla build template overrides file exists
+  template:
+    src: template-override.j2.j2
+    dest: "{{ kolla_build_config_path }}/template-override.j2"
     mode: 0644
-  with_items:
-    - { src: kolla-build.conf.j2, dest: kolla-build.conf }
-    - { src: template-override.j2.j2, dest: template-override.j2 }

ansible/roles/kolla-build/templates/kolla/kolla-build.conf

@@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 [DEFAULT]
 
 # Base container image distribution.

doc/source/multiple-environments.rst

@@ -70,7 +70,7 @@ Ansible Inventories
 Each environment can include its own inventory, which overrides any variable
 declaration done in the shared inventory. Typically, a shared inventory may be
 used to define groups and group variables, while hosts and host variables would
-be set in enviroment inventories. The following layout (ignoring non-inventory
+be set in environment inventories. The following layout (ignoring non-inventory
 files) shows an example of multiple inventories.
 
 .. code-block:: text
@@ -152,10 +152,31 @@ like the ``network-allocation.yml`` file.
 Kolla Configuration
 -------------------
 
-Kolla configuration is currently independent in each environment. To avoid
-duplicating configuration, symbolic links can be used to share common variable
-definitions. It is advised to avoid sharing credentials between environments by
-making each Kolla ``passwords.yml`` file unique.
+In the Wallaby release, Kolla configuration was independent in each
+environment.
+
+As of the Xena release, the following files support combining the
+environment-specific and shared configuration file content:
+
+* ``kolla/config/bifrost/bifrost.yml``
+* ``kolla/config/bifrost/dib.yml``
+* ``kolla/config/bifrost/servers.yml``
+* ``kolla/globals.yml``
+* ``kolla/kolla-build.conf``
+
+Options in the environment-specific files take precedence over those in the
+shared files.
+
+Managing Independent Environment Files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For files that are independent in each environment, i.e. they do not support
+combining the environment-specific and shared configuration file content, there
+are some techniques that may be used to avoid duplication.
+
+For example, symbolic links can be used to share common variable definitions.
+It is advised to avoid sharing credentials between environments by making each
+Kolla ``passwords.yml`` file unique.
 
 Custom Ansible Playbooks and Hooks
 ----------------------------------

kayobe/plugins/action/merge_yaml.py

@@ -86,7 +86,7 @@ class ActionModule(action.ActionBase):
     def read_config(self, source):
         result = None
         # Only use config if present
-        if os.access(source, os.R_OK):
+        if source and os.access(source, os.R_OK):
             with open(source, 'r') as f:
                 template_data = f.read()
 

releasenotes/notes/merge-configs-1f8fb3672e9be404.yaml

@@ -0,0 +1,22 @@
+---
+features:
+  - |
+    Adds support for merging the following configuration files from the
+    environment-specific directory (``etc/kayobe/environments/<environment>``)
+    and the base directory (``etc/kayobe``).
+
+    * ``kolla/config/bifrost/bifrost.yml``
+    * ``kolla/config/bifrost/dib.yml``
+    * ``kolla/config/bifrost/servers.yml``
+    * ``kolla/globals.yml``
+    * ``kolla/kolla-build.conf``
+
+    See `story 2002009 <https://storyboard.openstack.org/#!/story/2002009>`__
+    for details.
+deprecations:
+  - |
+    The following variables are deprecated, in favour of using configuration
+    files ``kolla/globals.yml`` and ``kolla/kolla-build.conf`` respectively.
+
+    * ``kolla_extra_globals``
+    * ``kolla_bifrost_extra_globals``

tools/test-ansible.sh

@@ -6,6 +6,7 @@
 set -e
 
 failed=0
+export ANSIBLE_ACTION_PLUGINS="kayobe/plugins/action:~/.ansible/plugins/action:/usr/share/ansible/plugins/action"
 for playbook in ansible/roles/*/tests/main.yml; do
     # We declare extra variables to install the {{ openstack_branch }} version
     # of kolla-ansible. We should use {{ kolla_ansible_source_version }}, but