Use merge_configs and merge_yaml to generate Kolla custom config

This patch adds new functionality - merging base & environment specific kolla config. This allows you to place common settings in the base configuration and only keep environment specific settings in the environment directories. Change-Id: Id4588f4529a4522e68e22ce58711cb927fa68a9d Story: 2002009 Task: 42903
2021-08-10 11:53:16 +01:00 · 2021-08-10 11:53:16 +01:00 · 428ef10fa4
commit 428ef10fa4
parent c75a32e72f
46 changed files with 1226 additions and 557 deletions
--- a/ansible/inventory/group_vars/all/kolla
+++ b/ansible/inventory/group_vars/all/kolla
@ -571,6 +571,16 @@ kolla_enable_vitrage: "no"
 kolla_enable_watcher: "no"
 kolla_enable_zun: "no"

+###############################################################################
+# Kolla custom config generation.
+
+# Feature flag to add $KAYOBE_CONFIG_PATH to the list of search paths used
+# when searching for Kolla custom service configuration. Only has an effect in
+# a multiple environments setup. This allows you to configure merging between
+# your environment and the base layer. Defaults to true. Set to false for
+# backwards compatibility.
+kolla_openstack_custom_config_environment_merging_enabled: true
+
 ###############################################################################
 # Passwords and credentials.

--- a/ansible/kolla-openstack.yml
+++ b/ansible/kolla-openstack.yml
@ -100,53 +100,6 @@
    ipa_image_name: "ipa"
  pre_tasks:
    - block:
-        - name: Check whether Kolla extra configuration files exist
-          stat:
-            path: "{{ kayobe_env_config_path }}/kolla/config/{{ item.file }}"
-            get_checksum: False
-            get_md5: False
-            mime: False
-          register: stat_result
-          with_items:
-            - { name: aodh, file: aodh.conf }
-            - { name: barbican, file: barbican.conf }
-            - { name: blazar, file: blazar.conf }
-            - { name: ceilometer, file: ceilometer.conf }
-            - { name: cinder, file: cinder.conf }
-            - { name: cloudkitty, file: cloudkitty.conf }
-            - { name: designate, file: designate.conf }
-            - { name: glance, file: glance.conf }
-            - { name: global, file: global.conf }
-            - { name: gnocchi, file: gnocchi.conf }
-            - { name: grafana, file: grafana.ini }
-            - { name: heat, file: heat.conf }
-            - { name: inspector, file: ironic-inspector.conf }
-            - { name: ironic, file: ironic.conf }
-            - { name: keystone, file: keystone.conf }
-            - { name: magnum, file: magnum.conf }
-            - { name: manila, file: manila.conf }
-            - { name: mariabackup, file: backup.my.cnf }
-            - { name: mariadb, file: galera.cnf }
-            - { name: masakari, file: masakari.conf }
-            - { name: multipathd, file: multipath.conf }
-            - { name: murano, file: murano.conf }
-            - { name: neutron, file: neutron.conf }
-            - { name: neutron_ml2, file: neutron/ml2_conf.ini }
-            - { name: nova, file: nova.conf }
-            - { name: octavia, file: octavia.conf }
-            - { name: placement, file: placement.conf }
-            - { name: sahara, file: sahara.conf }
-
-        - name: Initialise a fact containing extra configuration
-          set_fact:
-            kolla_extra_config: {}
-
-        - name: Update a fact containing extra configuration
-          set_fact:
-            kolla_extra_config: "{{ kolla_extra_config | combine({item.item.name: lookup('template', '{{ item.stat.path }}')}) }}"
-          with_items: "{{ stat_result.results }}"
-          when: item.stat.exists
-
        - name: Validate switch configuration for Neutron ML2 genericswitch driver
          fail:
            msg: >
@ -217,35 +170,11 @@
      kolla_inspector_swift_auth:
        auth_type: none
        endpoint_override: "http://{% raw %}{{ api_interface_address }}{% endraw %}:{{ inspector_store_port }}"
-      # Extra free-form user-provided configuration.
-      kolla_extra_aodh: "{{ kolla_extra_config.aodh | default }}"
-      kolla_extra_barbican: "{{ kolla_extra_config.barbican | default }}"
-      kolla_extra_blazar: "{{ kolla_extra_config.blazar | default }}"
-      kolla_extra_ceilometer: "{{ kolla_extra_config.ceilometer | default }}"
-      kolla_extra_cinder: "{{ kolla_extra_config.cinder | default }}"
-      kolla_extra_cloudkitty: "{{ kolla_extra_config.cloudkitty | default }}"
-      kolla_extra_designate: "{{ kolla_extra_config.designate | default }}"
-      kolla_extra_glance: "{{ kolla_extra_config.glance | default }}"
-      kolla_extra_global: "{{ kolla_extra_config.global | default }}"
-      kolla_extra_gnocchi: "{{ kolla_extra_config.gnocchi | default }}"
-      kolla_extra_grafana: "{{ kolla_extra_config.grafana | default }}"
-      kolla_extra_heat: "{{ kolla_extra_config.heat | default }}"
-      kolla_extra_inspector: "{{ kolla_extra_config.inspector | default }}"
-      kolla_extra_ironic: "{{ kolla_extra_config.ironic | default }}"
-      kolla_extra_keystone: "{{ kolla_extra_config.keystone | default }}"
-      kolla_extra_magnum: "{{ kolla_extra_config.magnum | default }}"
-      kolla_extra_manila: "{{ kolla_extra_config.manila | default }}"
-      kolla_extra_mariabackup: "{{ kolla_extra_config.mariabackup | default }}"
-      kolla_extra_mariadb: "{{ kolla_extra_config.mariadb | default }}"
-      kolla_extra_masakari: "{{ kolla_extra_config.masakari | default }}"
-      kolla_extra_multipathd: "{{ kolla_extra_config.multipathd | default }}"
-      kolla_extra_murano: "{{ kolla_extra_config.murano | default }}"
-      kolla_extra_neutron: "{{ kolla_extra_config.neutron | default }}"
-      kolla_extra_neutron_ml2: "{{ kolla_extra_config.neutron_ml2 | default }}"
-      kolla_extra_nova: "{{ kolla_extra_config.nova | default }}"
-      kolla_extra_octavia: "{{ kolla_extra_config.octavia | default }}"
-      kolla_extra_placement: "{{ kolla_extra_config.placement | default }}"
-      kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}"
-      kolla_extra_config_path: "{{ kayobe_env_config_path }}/kolla/config"
+      kolla_openstack_custom_config_paths_extra_multi_env:
+        - "{{ kayobe_config_path }}"
+        - "{{ kayobe_env_config_path }}"
+      kolla_openstack_custom_config_paths_extra_legacy:
+        - "{{ kayobe_env_config_path }}"
+      kolla_openstack_custom_config_paths_extra: "{{ kolla_openstack_custom_config_paths_extra_multi_env if kolla_openstack_custom_config_environment_merging_enabled | bool else kolla_openstack_custom_config_paths_extra_legacy }}"
      kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
      kolla_nova_libvirt_certificates_src: "{{ kayobe_env_config_path }}/certificates/libvirt"
--- a/ansible/roles/image-download/tasks/main.yml
+++ b/ansible/roles/image-download/tasks/main.yml
@ -1,4 +1,9 @@
 ---
+- name: Ensure destination directory exists
+  file:
+    state: directory
+    path: "{{ image_download_dest | dirname }}"
+
 - block:
    - block:
        - name: Fail if the checksum algorithm is not set
--- a/ansible/roles/kolla-openstack/action_plugins/kolla_custom_config_info.py
+++ b/ansible/roles/kolla-openstack/action_plugins/kolla_custom_config_info.py
@ -0,0 +1,229 @@
+# Copyright (c) 2023 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ansible.plugins.action import ActionBase
+import os
+from collections import defaultdict
+import pathlib
+
+from wcmatch import glob
+
+def _dedup(xs):
+    # Deduplicate a list whilst maintaining order
+    seen = set()
+    result = []
+    for x in xs:
+        if x not in seen:
+            seen.add(x)
+            result.append(x)
+    return result
+
+class ConfigCollector(object):
+    def __init__(self, include_globs, ignore_globs, destination, search_paths,
+                 rules):
+        # This variable groups together files in the search paths with
+        # the same relative path, for example if the search paths were:
+        # - {{ kayobe_config_env_path }}/
+        # - {{ kayobe_config_path }}/
+        # - {{ role_path }}/templates/
+        # and one of the include_globs matched nova.conf. You'd end up
+        # with the following files grouped together:
+        # - {{ kayobe_env_path }}/etc/kolla/nova.conf
+        # - {{ kayobe_config_path }}/etc/kolla/nova.conf
+        # - {{ role_path }}/templates/etc/kolla/nova.conf
+        # The key in the dictionary is the relative path of the file. The
+        # value is a list of absolute paths. This gets populated by the
+        # collect() method.
+        self.files_in_source = defaultdict(list)
+        # Set of files in destination. This is used to cleanup up files
+        # from a previous run that are no longer templated. Before any templating
+        # this variable is populated with all the files in the output directory.
+        # Each file that would be templated in the current run will be removed
+        # from this set as they are discovered.
+        self.files_in_destination = set()
+        # Determines which files are candidates for templating
+        self.include_globs = include_globs
+        # Some files are templated by external tasks. This is a list of files
+        # to not clean up.
+        self.ignore_globs = ignore_globs
+        # Where the files are being templated to
+        self.destination = destination
+        # Where to search for the source files
+        self.search_paths = search_paths
+
+        # Rules to determine merging strategy when multiple files are found
+        # with the same relative path. Lower priority numbers win.
+        self.rules = sorted(rules, key=lambda d: d['priority'])
+
+    def filter_files_in_destination(self):
+        ignored = set()
+        for f in self.files_in_destination:
+            for item in self.ignore_globs:
+                if not item["enabled"]:
+                    continue
+                if glob.globmatch(f, item["glob"], flags=glob.GLOBSTAR):
+                    ignored.add(f)
+        result = set(self.files_in_destination) - ignored
+        return list(result)
+
+    def _find_matching_rule(self, relative_path):
+        # First match wins
+        for rule in self.rules:
+            if not rule.get('enabled', True):
+                continue
+            glob_ = rule["glob"]
+            if glob.globmatch(relative_path, glob_, flags=glob.GLOBSTAR):
+                return rule
+
+    def partition_into_actions(self):
+        actions = {
+            "merge_yaml": [],
+            "merge_configs": [],
+            "template": [],
+            "copy": [],
+            "concat": [],
+            "create_dir": [],
+            "delete": []
+        }
+        missing_directories = set()
+        files_to_delete = self.filter_files_in_destination()
+
+        # Convert to absolute paths
+        files_to_delete = {
+            os.path.join(self.destination, x) for x in files_to_delete
+        }
+
+        for relative_path, sources in self.files_in_source.items():
+            found_match = False
+            destination = os.path.join(self.destination, relative_path)
+            # Don't delete any files we are templating
+            files_to_delete.discard(destination)
+
+            dirname = os.path.dirname(destination)
+            if not os.path.exists(dirname):
+                missing_directories.add(dirname)
+
+            rule = self._find_matching_rule(relative_path)
+
+            if not rule:
+                continue
+
+            if rule["strategy"] == 'copy':
+                copy = {
+                    "src": sources[-1],
+                    "dest": destination,
+                    "params": rule.get('params', [])
+                }
+                actions["copy"].append(copy)
+                continue
+
+            if rule["strategy"] == "merge_yaml":
+                merge_yaml = {
+                    "sources": sources,
+                    "dest": destination,
+                    "params": rule.get('params', [])
+                }
+                actions["merge_yaml"].append(merge_yaml)
+                continue
+
+            if rule["strategy"] == "merge_configs":
+                merge_configs = {
+                    "sources": sources,
+                    "dest": destination,
+                    "params": rule.get('params', [])
+                }
+                actions["merge_configs"].append(merge_configs)
+                continue
+
+            if rule["strategy"] == "concat":
+                concat = {
+                    "sources": sources,
+                    "dest": destination,
+                    "params": rule.get('params', [])
+                }
+                actions["concat"].append(concat)
+                continue
+
+            if rule["strategy"] == "template":
+                template = {
+                    "src": sources[-1],
+                    "dest": destination,
+                    "params": rule.get('params', [])
+                }
+                actions["template"].append(template)
+                continue
+
+        actions["create_dir"] = list(missing_directories)
+        # Sort by length so that subdirectories are created after the parent
+        actions["create_dir"].sort(key=len)
+
+        actions["delete"] = list(files_to_delete)
+        return actions
+
+    def collect(self):
+        for item in self.include_globs:
+            self._collect_source(item)
+            self._collect_destination(item)
+
+    def _collect_source(self, item):
+        enabled = item.get("enabled", False)
+        if not isinstance(enabled, bool):
+            raise ValueError("Expecting a boolean: %s" % item)
+        if not enabled:
+            return
+        for search_path in self.search_paths:
+            abs_glob = os.path.join(search_path, item["glob"])
+            files = glob.glob(abs_glob, flags=glob.GLOBSTAR)
+            for abs_path in files:
+                if not os.path.isfile(abs_path):
+                    continue
+                relative_path = os.path.relpath(abs_path, search_path)
+                self.files_in_source[relative_path].append(abs_path)
+
+    def _collect_destination(self, item):
+        abs_glob = os.path.join(self.destination, item["glob"])
+        files = glob.glob(abs_glob, flags=glob.GLOBSTAR)
+        for abs_path in files:
+            if not os.path.isfile(abs_path):
+                continue
+            relative_path = os.path.relpath(abs_path, self.destination)
+            self.files_in_destination.add(relative_path)
+
+class ActionModule(ActionBase):
+
+    def run(self, tmp=None, task_vars=None):
+        if task_vars is None:
+            task_vars = dict()
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+
+        # This class never changes anything. We only collect the extra config
+        # files and group by action.
+        result['changed'] = False
+
+        args = self._task.args
+
+        collector = ConfigCollector(
+            destination=args.get("destination"),
+            ignore_globs=args.get("ignore_globs"),
+            include_globs=args.get("include_globs"),
+            rules=args.get("rules"),
+            search_paths=_dedup(args["search_paths"])
+        )
+
+        collector.collect()
+
+        result.update(collector.partition_into_actions())
+
+        return result
--- a/ansible/roles/kolla-openstack/defaults/main.yml
+++ b/ansible/roles/kolla-openstack/defaults/main.yml
@ -1,6 +1,19 @@
 ---
-# Path to extra kolla-ansible configuration files.
-kolla_extra_config_path:
+# Ordered list of paths to default kolla-ansible configuration files. Least
+# specific first. Default is search the role templates in
+# templates/kolla/config.
+kolla_openstack_custom_config_paths_default:
+  - "{{ role_path }}/templates"
+
+# Ordered list of paths to extra kolla-ansible configuration files. Least
+# specific first. Default is an empty list.
+kolla_openstack_custom_config_paths_extra: []
+
+# Ordered list of paths to kolla-ansible configuration files. Least specific
+# first. Default is a combination of
+# kolla_openstack_custom_config_paths_default and
+# kolla_openstack_custom_config_paths_extra.
+kolla_openstack_custom_config_paths: "{{ kolla_openstack_custom_config_paths_default + kolla_openstack_custom_config_paths_extra }}"

 # Directory where Kolla custom configuration files will be installed.
 kolla_node_custom_config_path: /etc/kolla/config
@ -8,15 +21,307 @@ kolla_node_custom_config_path: /etc/kolla/config
 ###############################################################################
 # Global configuration.

+# Deprecated:
 # Free form extra configuration to append to global.conf.
 kolla_extra_global:

+###############################################################################
+# Kolla custom config generation.
+
+# Default value for kolla_openstack_custom_config_include_globs.
+kolla_openstack_custom_config_include_globs_default:
+  - enabled: '{{ kolla_enable_aodh | bool }}'
+    glob: aodh.conf
+  - enabled: '{{ kolla_enable_aodh | bool }}'
+    glob: aodh/**
+  - enabled: '{{ kolla_enable_barbican | bool }}'
+    glob: barbican.conf
+  - enabled: '{{ kolla_enable_barbican | bool }}'
+    glob: barbican/**
+  - enabled: '{{ kolla_enable_barbican | bool }}'
+    glob: barbican-api/**
+  - enabled: '{{ kolla_enable_blazar | bool }}'
+    glob: blazar.conf
+  - enabled: '{{ kolla_enable_blazar | bool }}'
+    glob: blazar/**
+  - enabled: '{{ kolla_enable_ceilometer | bool }}'
+    glob: ceilometer.conf
+  - enabled: '{{ kolla_enable_ceilometer | bool }}'
+    glob: ceilometer/**
+  - enabled: '{{ kolla_enable_cinder | bool }}'
+    glob: cinder.conf
+  - enabled: '{{ kolla_enable_cinder | bool }}'
+    glob: nfs_shares
+  - enabled: '{{ kolla_enable_cinder | bool }}'
+    glob: cinder/**
+  - enabled: '{{ kolla_enable_cloudkitty | bool }}'
+    glob: cloudkitty.conf
+  - enabled: '{{ kolla_enable_cloudkitty | bool }}'
+    glob: cloudkitty/**
+  - enabled: '{{ kolla_enable_designate | bool }}'
+    glob: designate.conf
+  - enabled: '{{ kolla_enable_designate | bool }}'
+    glob: designate/**
+  - enabled: '{{ kolla_enable_fluentd | bool }}'
+    glob: fluentd/**/*.conf
+  - enabled: '{{ kolla_enable_mariadb | bool }}'
+    glob: galera.cnf
+  - enabled: '{{ kolla_enable_glance | bool }}'
+    glob: glance*.conf
+  - enabled: '{{ kolla_enable_glance | bool }}'
+    glob: glance/**
+  - enabled: true
+    glob: global.conf
+  - enabled: '{{ kolla_enable_gnocchi | bool }}'
+    glob: gnocchi.conf
+  - enabled: '{{ kolla_enable_gnocchi | bool }}'
+    glob: gnocchi/**
+  - enabled: '{{ kolla_enable_grafana | bool }}'
+    glob: grafana.ini
+  - enabled: '{{ kolla_enable_grafana | bool }}'
+    glob: grafana/**
+  - enabled: '{{ kolla_enable_haproxy | bool }}'
+    glob: haproxy-config/**
+  - enabled: '{{ kolla_enable_haproxy | bool }}'
+    glob: haproxy/**
+  - enabled: '{{ kolla_enable_heat | bool }}'
+    glob: heat.conf
+  - enabled: '{{ kolla_enable_heat | bool }}'
+    glob: heat/**
+  - enabled: '{{ kolla_enable_horizon | bool }}'
+    glob: horizon/**
+  - enabled: '{{ kolla_enable_influxdb | bool }}'
+    glob: influx*
+  - enabled: '{{ kolla_enable_ironic | bool }}'
+    glob: ironic-inspector.conf
+  - enabled: '{{ kolla_enable_ironic | bool }}'
+    glob: ironic.conf
+  - enabled: '{{ kolla_enable_ironic | bool }}'
+    glob: ironic/**
+  - enabled: '{{ kolla_enable_keepalived | bool }}'
+    glob: keepalived/**
+  - enabled: '{{ kolla_enable_keystone | bool }}'
+    glob: keystone.conf
+  - enabled: '{{ kolla_enable_keystone | bool }}'
+    glob: keystone/**
+  - enabled: true
+    glob: kolla-toolbox/**
+  - enabled: '{{ kolla_enable_magnum | bool }}'
+    glob: magnum.conf
+  - enabled: '{{ kolla_enable_magnum | bool }}'
+    glob: magnum/**
+  - enabled: '{{ kolla_enable_manila | bool }}'
+    glob: manila.conf
+  - enabled: '{{ kolla_enable_manila | bool }}'
+    glob: manila/**
+  - enabled: '{{ kolla_enable_mariadb | bool }}'
+    glob: backup.my.cnf
+  - enabled: '{{ kolla_enable_mariadb | bool }}'
+    glob: mariadb/**
+  - enabled: '{{ kolla_enable_masakari | bool }}'
+    glob: masakari.conf
+  - enabled: '{{ kolla_enable_masakari | bool }}'
+    glob: masakari/**
+  - enabled: '{{ kolla_enable_multipathd | bool }}'
+    glob: multipath.conf
+  - enabled: '{{ kolla_enable_multipathd | bool }}'
+    glob: multipath/**
+  - enabled: '{{ kolla_enable_murano | bool }}'
+    glob: murano.conf
+  - enabled: '{{ kolla_enable_murano | bool }}'
+    glob: murano/**
+  - enabled: '{{ kolla_enable_neutron | bool }}'
+    glob: neutron.conf
+  - enabled: '{{ kolla_enable_neutron | bool }}'
+    glob: neutron/**
+  - enabled: '{{ kolla_enable_nova | bool }}'
+    glob: nova.conf
+  - enabled: '{{ kolla_enable_nova | bool }}'
+    glob: nova/**
+  - enabled: '{{ kolla_enable_nova | bool }}'
+    glob: nova_compute/**
+  - enabled: '{{ kolla_enable_octavia | bool  }}'
+    glob: octavia.conf
+  - enabled: '{{ kolla_enable_octavia | bool }}'
+    glob: octavia/**
+  - enabled: '{{ kolla_enable_opensearch | bool }}'
+    glob: opensearch.yml
+  - enabled: '{{ kolla_enable_opensearch | bool }}'
+    glob: opensearch/**
+  - enabled: '{{ kolla_enable_placement | bool }}'
+    glob: placement.conf
+  - enabled: '{{ kolla_enable_placement | bool }}'
+    glob: placement/**
+  - enabled: '{{ kolla_enable_prometheus | bool }}'
+    glob: prometheus/**
+  - enabled: '{{ kolla_enable_sahara | bool }}'
+    glob: sahara.conf
+  - enabled: '{{ kolla_enable_sahara | bool }}'
+    glob: sahara/**
+  - enabled: '{{ kolla_enable_swift | bool }}'
+    glob: swift/**
+
+# Extra items to add to kolla_openstack_custom_config_include_globs_default
+# to produce kolla_openstack_custom_config_include_globs.
+kolla_openstack_custom_config_include_globs_extra: []
+
+# List of dictionaries with the following keys:
+#   glob: a glob pattern. Any files matching this pattern will be copied to the
+#         the kolla custom config directory
+#   enabled: boolean to disable the glob.
+# This determines the list of files to copy to the generated kolla config
+# directory.
+kolla_openstack_custom_config_include_globs: "{{
+  kolla_openstack_custom_config_include_globs_default +
+  kolla_openstack_custom_config_include_globs_extra }}"
+
+# Kolla config generation rules. These operate on the list of files produced by
+# applying kolla_openstack_custom_config_include_globs. Each of the paths in
+# kolla_openstack_custom_config_paths is searched for files matching one of the
+# globs. If a match is found, any files with the same relative path are grouped
+# together. The rules determine what to do with these matching files e.g copy
+# the most specific file without templating, merge the files with
+# merge_configs, etc.
+# List of dictionaries with the following keys:
+#   glob: A glob matching files for this rule to match on (relative to the
+#     search path)
+#   priority: The rules are processed in increasing priority order with the
+#     first rule matching taking effect.
+#   strategy: How to process the matched file. One of copy, concat, template,
+#      merge_configs, merge_yaml
+#   params: List of params to pass to module enacting the strategy
+# Strategies:
+#   copy: Copy most specific file to kolla config without templating
+#   template: Template most specific file to kolla config
+#   concat: Concatenate files and copy the result to generated kolla config
+#   merge_configs: Use the merge_configs module to merge an ini file, before
+#     copying to the generated kolla-config.
+#   merge_yaml: Use the merge_yaml module to merge a file, before copying to
+#     the generated kolla-config.
+kolla_openstack_custom_config_rules: "{{ kolla_openstack_custom_config_rules_default | rejectattr('glob', 'in', kolla_openstack_custom_config_rules_default_remove) + kolla_openstack_custom_config_rules_extra }}"
+
+# Whether to enable ini merging rules in
+# kolla_openstack_custom_config_rules_default. Default is true.
+kolla_openstack_custom_config_merge_configs_enabled: true
+
+# Whether to enable yaml merging rules in
+# kolla_openstack_custom_config_rules_default. Default is true.
+kolla_openstack_custom_config_merge_yaml_enabled: true
+
+# Default merge strategy for ini files in
+# kolla_openstack_custom_config_rules_default. Default is concat.
+kolla_openstack_custom_config_ini_merge_strategy_default: concat
+
+# Default value for kolla_openstack_custom_config_rules.
+kolla_openstack_custom_config_rules_default:
+  - glob: horizon/themes/**
+    strategy: copy
+    priority: 1000
+  - glob: ironic/ironic-agent.initramfs
+    strategy: copy
+    priority: 1000
+  - glob: ironic/ironic-agent.kernel
+    strategy: copy
+    priority: 1000
+  - glob: swift/*.builder
+    strategy: copy
+    priority: 1000
+  - glob: swift/*.ring.gz
+    strategy: copy
+    priority: 1000
+  - glob: '**/*.pem'
+    strategy: copy
+    priority: 1000
+  # Exceptions for *.conf files which are not INI format
+  - glob: "**/collectd.conf"
+    strategy: template
+    priority: 1000
+  - glob: designate/**/named.conf
+    strategy: template
+    priority: 1000
+  - glob: designate/**/rndc.conf
+    strategy: template
+    priority: 1000
+  - glob: "**/dnsmasq.conf"
+    strategy: template
+    priority: 1000
+  - glob: fluentd/**/*.conf
+    strategy: template
+    priority: 1000
+  - glob: hacluster-corosync/**/corosync.conf
+    strategy: template
+    priority: 1000
+  - glob: horizon/**/horizon.conf
+    strategy: template
+    priority: 1000
+  - glob: "**/*httpd.conf"
+    strategy: template
+    priority: 1000
+  - glob: "**/influxdb.conf"
+    strategy: template
+    priority: 1000
+  - glob: "**/keepalived.conf"
+    strategy: template
+    priority: 1000
+  - glob: "**/multipath.conf"
+    strategy: template
+    priority: 1000
+  - glob: "**/rabbitmq*.conf"
+    strategy: template
+    priority: 1000
+  - glob: "**/*wsgi*.conf"
+    strategy: template
+    priority: 1000
+  # INI files
+  - glob: "**/*.conf"
+    strategy: "{{ kolla_openstack_custom_config_ini_merge_strategy_default }}"
+    priority: 2000
+    enabled: "{{ kolla_openstack_custom_config_merge_configs_enabled | bool }}"
+  - glob: "**/*.ini"
+    strategy: "{{ kolla_openstack_custom_config_ini_merge_strategy_default }}"
+    priority: 2000
+    enabled: "{{ kolla_openstack_custom_config_merge_configs_enabled | bool }}"
+  - glob: "**/galera.cnf"
+    strategy: "{{ kolla_openstack_custom_config_ini_merge_strategy_default }}"
+    priority: 2000
+    enabled: "{{ kolla_openstack_custom_config_merge_configs_enabled | bool }}"
+  - glob: "**/kafka.server.properties"
+    strategy: "{{ kolla_openstack_custom_config_ini_merge_strategy_default }}"
+    priority: 2000
+    enabled: "{{ kolla_openstack_custom_config_merge_configs_enabled | bool }}"
+  - glob: "**/*my.cnf"
+    strategy: "{{ kolla_openstack_custom_config_ini_merge_strategy_default }}"
+    priority: 2000
+    enabled: "{{ kolla_openstack_custom_config_merge_configs_enabled | bool }}"
+  # YAML files
+  - glob: "**/*.yml"
+    strategy: merge_yaml
+    priority: 2000
+    enabled: "{{ kolla_openstack_custom_config_merge_yaml_enabled | bool }}"
+  - glob: "**/*.yaml"
+    strategy: merge_yaml
+    priority: 2000
+    enabled: "{{ kolla_openstack_custom_config_merge_yaml_enabled | bool }}"
+  # Catch all. Fallback to templating to match legacy behaviour.
+  - glob: '**'
+    strategy: template
+    priority: 65535
+
+# List of globs to filter from kolla_openstack_custom_config_rules_default.
+# Default is an empty list.
+kolla_openstack_custom_config_rules_default_remove: []
+
+# Extra items to add to kolla_openstack_custom_config_rules_default
+# to produce kolla_openstack_custom_config_rules.
+kolla_openstack_custom_config_rules_extra: []
+
 ###############################################################################
 # Aodh configuration.

 # Whether to enable Aodh.
-kolla_enable_aodh:
+kolla_enable_aodh: false

+# Deprecated:
 # Free form extra configuration to append to aodh.conf.
 kolla_extra_aodh:

@ -24,8 +329,9 @@ kolla_extra_aodh:
 # Barbican configuration.

 # Whether to enable Barbican.
-kolla_enable_barbican:
+kolla_enable_barbican: false

+# Deprecated:
 # Free form extra configuration to append to barbican.conf.
 kolla_extra_barbican:

@ -33,8 +339,9 @@ kolla_extra_barbican:
 # Blazar configuration.

 # Whether to enable Blazar.
-kolla_enable_blazar:
+kolla_enable_blazar: false

+# Deprecated:
 # Free form extra configuration to append to blazar.conf.
 kolla_extra_blazar:

@ -42,8 +349,9 @@ kolla_extra_blazar:
 # Ceilometer configuration.

 # Whether to enable Ceilometer.
-kolla_enable_ceilometer:
+kolla_enable_ceilometer: false

+# Deprecated:
 # Free form extra configuration to append to ceilometer.conf.
 kolla_extra_ceilometer:

@ -51,8 +359,9 @@ kolla_extra_ceilometer:
 # cinder configuration.

 # Whether to enable cinder.
-kolla_enable_cinder:
+kolla_enable_cinder: false

+# Deprecated:
 # Free form extra configuration to append to cinder.conf.
 kolla_extra_cinder:

@ -60,8 +369,9 @@ kolla_extra_cinder:
 # CloudKitty configuration.

 # Whether to enable CloudKitty.
-kolla_enable_cloudkitty:
+kolla_enable_cloudkitty: false

+# Deprecated:
 # Free form extra configuration to append to cloudkitty.conf.
 kolla_extra_cloudkitty:

@ -69,17 +379,25 @@ kolla_extra_cloudkitty:
 # designate configuration.

 # Whether to enable designate.
-kolla_enable_designate:
+kolla_enable_designate: false

+# Deprecated:
 # Free form extra configuration to append to designate.conf.
 kolla_extra_designate:

+###############################################################################
+# Fluentd configuration.
+
+# Whether to enable Fluentd.
+kolla_enable_fluentd: false
+
 ###############################################################################
 # Glance configuration.

 # Whether to enable Glance.
-kolla_enable_glance:
+kolla_enable_glance: false

+# Deprecated:
 # Free form extra configuration to append to glance-api.conf and
 # glance-registry.conf.
 kolla_extra_glance:
@ -88,8 +406,9 @@ kolla_extra_glance:
 # Gnocchi configuration.

 # Whether to enable Gnocchi.
-kolla_enable_gnocchi:
+kolla_enable_gnocchi: false

+# Deprecated:
 # Free form extra configuration to append to gnocchi.conf.
 kolla_extra_gnocchi:

@ -97,11 +416,12 @@ kolla_extra_gnocchi:
 # Grafana configuration.

 # Whether to enable Grafana.
-kolla_enable_grafana:
+kolla_enable_grafana: false

 # Name of the admin user for Grafana.
 grafana_local_admin_user_name:

+# Deprecated:
 # Free form extra configuration to append to grafana.ini.
 kolla_extra_grafana:

@ -109,23 +429,15 @@ kolla_extra_grafana:
 # HAProxy configuration.

 # Whether to enable HAProxy.
-kolla_enable_haproxy:
-
-###############################################################################
-# Keystone configuration.
-
-# Whether to enable Keystone.
-kolla_enable_keystone:
-
-# Free form extra configuration to append to Keystone.conf
-kolla_extra_keystone:
+kolla_enable_haproxy: false

 ##############################################################################
 # Heat configuration.

 # Whether to enable Heat.
-kolla_enable_heat:
+kolla_enable_heat: false

+# Deprecated:
 # Free form extra configuration to append to heat.conf.
 kolla_extra_heat:

@ -133,19 +445,19 @@ kolla_extra_heat:
 # Horizon configuration.

 # Whether to enable Horizon.
-kolla_enable_horizon:
+kolla_enable_horizon: false

 ###############################################################################
 # InfluxDB configuration.

 # Whether to enable InfluxDB.
-kolla_enable_influxdb:
+kolla_enable_influxdb: false

 ###############################################################################
 # Ironic configuration.

 # Whether to enable Ironic.
-kolla_enable_ironic:
+kolla_enable_ironic: false

 # List of enabled Ironic drivers.
 kolla_ironic_drivers:
@ -252,6 +564,7 @@ kolla_ironic_provisioning_network:
 # List of additional append parameters for baremetal PXE boot.
 kolla_ironic_pxe_append_params: []

+# Deprecated:
 # Free form extra configuration to append to ironic.conf.
 kolla_extra_ironic:

@ -313,15 +626,33 @@ kolla_inspector_enable_swift:
 # store.
 kolla_inspector_swift_auth: {}

+# Deprecated:
 # Free form extra configuration to append to ironic-inspector.conf.
 kolla_extra_inspector:

+###############################################################################
+# Keepalived configuration.
+
+# Whether to enable Keepalived.
+kolla_enable_keepalived: false
+
+###############################################################################
+# Keystone configuration.
+
+# Whether to enable Keystone.
+kolla_enable_keystone: false
+
+# Deprecated:
+# Free form extra configuration to append to Keystone.conf
+kolla_extra_keystone:
+
 ###############################################################################
 # Magnum configuration.

 # Whether to enable Magnum.
-kolla_enable_magnum:
+kolla_enable_magnum: false

+# Deprecated:
 # Free form extra configuration to append to magnum.conf.
 kolla_extra_magnum:

@ -329,8 +660,9 @@ kolla_extra_magnum:
 # Mariabackup configuration.

 # Whether to enable Mariabackup.
-kolla_enable_mariabackup:
+kolla_enable_mariabackup: false

+# Deprecated:
 # Free form extra configuration to append to backup.my.cnf.
 kolla_extra_mariabackup:

@ -338,8 +670,9 @@ kolla_extra_mariabackup:
 # MariaDB configuration.

 # Whether to enable MariaDB.
-kolla_enable_mariadb:
+kolla_enable_mariadb: false

+# Deprecated:
 # Free form extra configuration to append to galera.cnf.
 kolla_extra_mariadb:

@ -347,14 +680,19 @@ kolla_extra_mariadb:
 # Manila configuration.

 # Whether to enable Manila.
-kolla_enable_manila:
+kolla_enable_manila: false
+
+# Deprecated:
+# Free form extra configuration to append to manila.conf.
+kolla_extra_manila:

 ###############################################################################
 # Masakari configuration.

 # Whether to enable Masakari.
-kolla_enable_masakari:
+kolla_enable_masakari: false

+# Deprecated:
 # Free form extra configuration to append to masakari.conf.
 kolla_extra_masakari:

@ -362,7 +700,7 @@ kolla_extra_masakari:
 # Multipathd configuration.

 # Whether to enable Multipathd.
-kolla_enable_multipathd:
+kolla_enable_multipathd: false

 # Free form extra configuration to append to multipath.conf.
 kolla_extra_multipathd:
@ -371,8 +709,9 @@ kolla_extra_multipathd:
 # Murano configuration.

 # Whether to enable Murano.
-kolla_enable_murano:
+kolla_enable_murano: false

+# Deprecated:
 # Free form extra configuration to append to murano.conf.
 kolla_extra_murano:

@ -380,7 +719,7 @@ kolla_extra_murano:
 # Neutron configuration.

 # Whether to enable Neutron.
-kolla_enable_neutron:
+kolla_enable_neutron: false

 # List of Neutron ML2 mechanism drivers to use.
 kolla_neutron_ml2_mechanism_drivers: []
@ -416,9 +755,11 @@ kolla_neutron_ml2_generic_switches: []
 # secret: not currently supported
 kolla_neutron_ml2_generic_switch_hosts: []

+# Deprecated:
 # Free form extra configuration to append to neutron.conf.
 kolla_extra_neutron:

+# Deprecated:
 # Free form extra configuration to append to ml2_conf.ini.
 kolla_extra_neutron_ml2:

@ -426,11 +767,12 @@ kolla_extra_neutron_ml2:
 # Nova configuration.

 # Whether to enable Nova.
-kolla_enable_nova:
+kolla_enable_nova: false

 # Whether to enable Nova libvirt container.
 kolla_enable_nova_libvirt_container:

+# Deprecated:
 # Free form extra configuration to append to nova.conf.
 kolla_extra_nova:

@ -441,24 +783,29 @@ kolla_libvirt_tls:
 # libvirt on the host.
 kolla_nova_libvirt_certificates_src:

+###############################################################################
+# OpenSearch configuration.
+
+# Whether to enable OpenSearch.
+kolla_enable_opensearch: false
+
 ###############################################################################
 # Octavia configuration.

 # Whether to enable Octavia.
-kolla_enable_octavia:
+kolla_enable_octavia: false

-###############################################################################
-# OpenSearch configuration.
-
-# Whether to enable opensearch.
-kolla_enable_opensearch:
+# Deprecated:
+# Free form extra configuration to append to octavia.conf
+kolla_extra_octavia:

 ###############################################################################
 # Placement configuration.

 # Whether to enable placement.
-kolla_enable_placement:
+kolla_enable_placement: false

+# Deprecated:
 # Free form extra configuration to append to placement.conf.
 kolla_extra_placement:

@ -466,14 +813,15 @@ kolla_extra_placement:
 # Prometheus configuration.

 # Whether to enable Prometheus.
-kolla_enable_prometheus:
+kolla_enable_prometheus: false

 ###############################################################################
 # Sahara configuration.

 # Whether to enable sahara.
-kolla_enable_sahara:
+kolla_enable_sahara: false

+# Deprecated:
 # Free form extra configuration to append to sahara.conf.
 kolla_extra_sahara:

@ -481,4 +829,4 @@ kolla_extra_sahara:
 # Swift configuration.

 # Whether to enable swift.
-kolla_enable_swift:
+kolla_enable_swift: false
--- a/ansible/roles/kolla-openstack/molecule/default/tests/test_default.py
+++ b/ansible/roles/kolla-openstack/molecule/default/tests/test_default.py
@ -25,22 +25,15 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')


-@pytest.mark.parametrize(
-    'path',
-    ['fluentd/filter',
-     'fluentd/input',
-     'fluentd/output'])
-def test_service_config_directory(host, path):
-    path = os.path.join('/etc/kolla/config', path)
-    utils.test_directory(host, path)
-
-
@pytest.mark.parametrize(
    'path',
    ['aodh',
     'cinder',
     'cloudkitty',
     'designate',
+     'fluentd/filter',
+     'fluentd/input',
+     'fluentd/output',
     'glance',
     'grafana',
     'heat',
--- a/ansible/roles/kolla-openstack/molecule/enable-everything/destroy.yml
+++ b/ansible/roles/kolla-openstack/molecule/enable-everything/destroy.yml
@ -25,3 +25,8 @@
      until: docker_jobs.finished
      retries: 300
      with_items: "{{ server.results }}"
+
+    - name: Clean up temporary path
+      file:
+        path: "{{ lookup('env', 'MOLECULE_TEMP_PATH') | default('/tmp/molecule', true) }}"
+        state: absent
--- a/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml
+++ b/ansible/roles/kolla-openstack/molecule/enable-everything/molecule.yml
@ -15,7 +15,16 @@ provisioner:
  inventory:
    group_vars:
      all:
-        kolla_extra_config_path: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/kolla/config
+        kolla_extra_config_path: ${MOLECULE_TEMP_PATH:-/tmp/molecule}/kolla/config
+        kolla_openstack_custom_config_paths_extra:
+          - "{{ kolla_extra_config_path }}/../.."
+        kolla_openstack_custom_config_rules_extra:
+          - glob: aodh/dummy.yml
+            strategy: merge_yaml
+            priority: 1000
+          - glob: aodh/dummy.ini
+            strategy: merge_configs
+            priority: 1000
        kolla_enable_aodh: true
        kolla_extra_aodh: |
          [extra-aodh.conf]
@ -44,6 +53,7 @@ provisioner:
        kolla_extra_designate: |
          [extra-designate.conf]
          foo=bar
+        kolla_enable_fluentd: true
        kolla_enable_glance: true
        kolla_extra_glance: |
          [extra-glance.conf]
@ -72,8 +82,9 @@ provisioner:
        kolla_extra_inspector: |
          [extra-ironic-inspector.conf]
          foo=bar
-        kolla_inspector_ipa_kernel_path: ${MOLECULE_TEMP_PATH:-/tmp}/ironic-agent.kernel
-        kolla_inspector_ipa_ramdisk_path: ${MOLECULE_TEMP_PATH:-/tmp}/ironic-agent.initramfs
+        kolla_inspector_ipa_kernel_path: ${MOLECULE_TEMP_PATH:-/tmp/molecule}/ironic-agent.kernel
+        kolla_inspector_ipa_ramdisk_path: ${MOLECULE_TEMP_PATH:-/tmp/molecule}/ironic-agent.initramfs
+        kolla_enable_keepalived: true
        kolla_enable_keystone: true
        kolla_extra_keystone: |
          [extra-keystone.conf]
@ -119,7 +130,7 @@ provisioner:
          [extra-nova.conf]
          foo=bar
        kolla_libvirt_tls: true
-        kolla_nova_libvirt_certificates_src: ${MOLECULE_TEMP_PATH:-/tmp}/molecule/nova-libvirt/certificates
+        kolla_nova_libvirt_certificates_src: ${MOLECULE_TEMP_PATH:-/tmp/molecule}/nova-libvirt/certificates
        kolla_enable_octavia: true
        kolla_extra_octavia: |
          [extra-octavia.conf]
--- a/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml
+++ b/ansible/roles/kolla-openstack/molecule/enable-everything/prepare.yml
@ -26,6 +26,99 @@
        - "{{ kolla_inspector_ipa_kernel_path }}"
        - "{{ kolla_inspector_ipa_ramdisk_path }}"

+    - name: Ensure parent directories of configuration files exist
+      file:
+        path: "{{ kolla_extra_config_path }}/{{ item }}"
+        state: directory
+        recurse: yes
+      delegate_to: localhost
+      run_once: true
+      with_items:
+        - neutron
+        - aodh
+        # To check that subdirectories are handled correctly
+        - prometheus/prometheus.yml.d
+        # Example of non-ini files that should be templated but not merged
+        - fluentd/input/
+
+    - name: Ensure extra INI configuration files exist
+      copy:
+        content: |
+          [extra-file-{{ item | basename }}]
+          bar=baz
+        dest: "{{ kolla_extra_config_path }}/{{ item }}"
+      run_once: true
+      delegate_to: localhost
+      loop:
+        - aodh.conf
+        - barbican.conf
+        - blazar.conf
+        - ceilometer.conf
+        - cinder.conf
+        - cloudkitty.conf
+        - designate.conf
+        - glance.conf
+        - global.conf
+        - gnocchi.conf
+        - grafana.ini
+        - heat.conf
+        - ironic.conf
+        - ironic-inspector.conf
+        - keystone.conf
+        - magnum.conf
+        - manila.conf
+        - murano.conf
+        - backup.my.cnf
+        - galera.cnf
+        - masakari.conf
+        - neutron.conf
+        - neutron/ml2_conf.ini
+        - nova.conf
+        - octavia.conf
+        - sahara.conf
+        - placement.conf
+
+    - name: Ensure extra YAML configuration files exist
+      copy:
+        content: |
+          dummy_variable: 123
+        dest: "{{ kolla_extra_config_path }}/{{ item }}"
+      run_once: true
+      delegate_to: localhost
+      loop:
+        - aodh/dummy.yml
+        - opensearch.yml
+        - prometheus/prometheus.yml.d/dummy.yml
+
+    - name: Template extra custom config files
+      # These correspond to globs defined in molecule.yml
+      copy:
+        content: "{{ item.content }}"
+        dest: "{{ kolla_extra_config_path }}/{{ item.dest }}"
+      run_once: true
+      delegate_to: localhost
+      with_items:
+        - dest: aodh/dummy.ini
+          content: |
+            [dummy-section]
+            dummy_variable = 123
+        - dest: fluentd/input/01-test.conf
+          content: |
+            <source>
+              @type tail
+              path /grepme
+              pos_file /var/run/td-agent/rabbit.pos
+              tag infra.rabbit
+              enable_watch_timer false
+              <parse>
+                @type multiline
+                format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}/
+                format1 /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \[(?<log_level>\w+)\] (?<Payload>.*)/
+              </parse>
+            </source>
+      loop_control:
+        label: "{{ item.dest }}"
+
    - name: Ensure nova libvirt certificates directory exists
      local_action:
        module: file
--- a/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py
+++ b/ansible/roles/kolla-openstack/molecule/enable-everything/tests/test_default.py
@ -19,46 +19,12 @@ from kayobe.tests.molecule import utils

 import pytest
 import testinfra.utils.ansible_runner
+import yaml


 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')

-
-@pytest.mark.parametrize(
-    'path',
-    ['aodh',
-     'barbican',
-     'cinder',
-     'cloudkitty',
-     'designate',
-     'fluentd/filter',
-     'fluentd/input',
-     'fluentd/output',
-     'glance',
-     'grafana',
-     'heat',
-     'horizon',
-     'ironic',
-     'keystone',
-     'magnum',
-     'manila',
-     'mariadb',
-     'masakari',
-     'murano',
-     'neutron',
-     'nova',
-     'nova/nova-libvirt',
-     'octavia',
-     'placement',
-     'prometheus',
-     'sahara',
-     'swift'])
-def test_service_config_directory(host, path):
-    path = os.path.join('/etc/kolla/config', path)
-    utils.test_directory(host, path)
-
-
@pytest.mark.parametrize(
    'path',
    ['aodh.conf',
@ -87,11 +53,44 @@ def test_service_config_directory(host, path):
     'backup.my.cnf'])
 def test_service_ini_file(host, path):
    # TODO(mgoddard): Check more of config file contents.
+    # Tests config added with extra vars e.g kolla_extra_aodh. I.e the
+    # the internal role templates.
    path = os.path.join('/etc/kolla/config', path)
    extra_section = 'extra-%s' % os.path.basename(path)
    expected = {extra_section: {'foo': 'bar'}}
    utils.test_ini_file(host, path, expected=expected)

+@pytest.mark.parametrize(
+    'path',
+    ['aodh.conf',
+     'barbican.conf',
+     'cinder.conf',
+     'cloudkitty.conf',
+     'designate.conf',
+     'galera.cnf',
+     'glance.conf',
+     'grafana.ini',
+     'heat.conf',
+     'ironic.conf',
+     'ironic-inspector.conf',
+     'keystone.conf',
+     'magnum.conf',
+     'manila.conf',
+     'masakari.conf',
+     'murano.conf',
+     'neutron/ml2_conf.ini',
+     'neutron.conf',
+     'nova.conf',
+     'octavia.conf',
+     'placement.conf',
+     'sahara.conf',
+     'backup.my.cnf'])
+def test_service_ini_file_extra_confs(host, path):
+    # Tests config added via extra config files
+    path = os.path.join('/etc/kolla/config', path)
+    extra_section = 'extra-file-%s' % os.path.basename(path)
+    expected = {extra_section: {'bar': 'baz'}}
+    utils.test_ini_file(host, path, expected=expected)

@pytest.mark.parametrize(
    'path',
@ -104,3 +103,30 @@ def test_service_non_ini_file(host, path):
    # TODO(mgoddard): Check config file contents.
    path = os.path.join('/etc/kolla/config', path)
    utils.test_file(host, path)
+
+@pytest.mark.parametrize(
+    'path,regex',
+    [('fluentd/input/01-test.conf', 'grepme')])
+def test_service_non_ini_file_regex(host, path, regex):
+    path = os.path.join('/etc/kolla/config', path)
+    utils.test_regex_in_file(host, path, regex=regex)
+
+@pytest.mark.parametrize(
+    'relative_path',
+    ['aodh/dummy.yml',
+     'opensearch.yml',
+     'prometheus/prometheus.yml.d/dummy.yml'])
+def test_service_extra_yml_config(host, relative_path):
+    path = os.path.join('/etc/kolla/config', relative_path)
+    utils.test_file(host, path)
+    content = yaml.safe_load(host.file(path).content_string)
+    assert content["dummy_variable"] == 123
+
+def test_service_extra_ini_config(host):
+    relative_path = "aodh/dummy.ini"
+    path = os.path.join('/etc/kolla/config', relative_path)
+    utils.test_file(host, path)
+    expected = {
+        "dummy-section": {"dummy_variable": "123"}
+    }
+    utils.test_ini_file(host, path, expected=expected)
--- a/ansible/roles/kolla-openstack/tasks/config.yml
+++ b/ansible/roles/kolla-openstack/tasks/config.yml
@ -1,48 +1,4 @@
 ---
- name: Ensure the Kolla OpenStack configuration directories exist
-  file:
-    path: "{{ item.dest }}"
-    state: directory
-    mode: 0750
-  with_items: "{{ kolla_openstack_custom_config }}"
-  when: item.enabled | bool
-
- name: Ensure the Kolla OpenStack configuration files exist
-  template:
-    src: "{{ item.src }}"
-    dest: "{{ kolla_node_custom_config_path }}/{{ item.dest }}"
-    mode: 0640
-  with_items:
-    - { src: aodh.conf.j2, dest: aodh.conf, enabled: "{{ kolla_enable_aodh }}" }
-    - { src: barbican.conf.j2, dest: barbican.conf, enabled: "{{ kolla_enable_barbican }}" }
-    - { src: blazar.conf.j2, dest: blazar.conf, enabled: "{{ kolla_enable_blazar }}" }
-    - { src: ceilometer.conf.j2, dest: ceilometer.conf, enabled: "{{ kolla_enable_ceilometer }}" }
-    - { src: cinder.conf.j2, dest: cinder.conf, enabled: "{{ kolla_enable_cinder }}" }
-    - { src: cloudkitty.conf.j2, dest: cloudkitty.conf, enabled: "{{ kolla_enable_cloudkitty }}" }
-    - { src: designate.conf.j2, dest: designate.conf, enabled: "{{ kolla_enable_designate }}" }
-    - { src: galera.cnf.j2, dest: galera.cnf, enabled: "{{ kolla_enable_mariadb }}" }
-    - { src: glance.conf.j2, dest: glance.conf, enabled: "{{ kolla_enable_glance }}" }
-    - { src: global.conf.j2, dest: global.conf, enabled: true }
-    - { src: gnocchi.conf.j2, dest: gnocchi.conf, enabled: "{{ kolla_enable_gnocchi }}" }
-    - { src: grafana.ini.j2, dest: grafana.ini, enabled: "{{ kolla_enable_grafana }}" }
-    - { src: heat.conf.j2, dest: heat.conf, enabled: "{{ kolla_enable_heat }}" }
-    - { src: ironic.conf.j2, dest: ironic.conf, enabled: "{{ kolla_enable_ironic }}" }
-    - { src: ironic-inspector.conf.j2, dest: ironic-inspector.conf, enabled: "{{ kolla_enable_ironic }}" }
-    - { src: keystone.conf.j2, dest: keystone.conf, enabled: "{{ kolla_enable_keystone }}" }
-    - { src: magnum.conf.j2, dest: magnum.conf, enabled: "{{ kolla_enable_magnum }}" }
-    - { src: manila.conf.j2, dest: manila.conf, enabled: "{{ kolla_enable_manila }}" }
-    - { src: backup.my.cnf.j2, dest: backup.my.cnf, enabled: "{{ kolla_enable_mariabackup }}" }
-    - { src: masakari.conf.j2, dest: masakari.conf, enabled: "{{ kolla_enable_masakari }}" }
-    - { src: ml2_conf.ini.j2, dest: neutron/ml2_conf.ini, enabled: "{{ kolla_enable_neutron }}" }
-    - { src: multipath.conf.j2, dest: multipath.conf, enabled: "{{ kolla_enable_multipathd }}" }
-    - { src: murano.conf.j2, dest: murano.conf, enabled: "{{ kolla_enable_murano }}" }
-    - { src: neutron.conf.j2, dest: neutron.conf, enabled: "{{ kolla_enable_neutron }}" }
-    - { src: nova.conf.j2, dest: nova.conf, enabled: "{{ kolla_enable_nova }}" }
-    - { src: octavia.conf.j2, dest: octavia.conf, enabled: "{{ kolla_enable_octavia }}" }
-    - { src: placement.conf.j2, dest: placement.conf, enabled: "{{ kolla_enable_placement }}" }
-    - { src: sahara.conf.j2, dest: sahara.conf, enabled: "{{ kolla_enable_sahara }}" }
-  when: item.enabled | bool
-
 - name: Ensure ironic inspector kernel and ramdisk images are present
  vars:
    image_download_url: "{{ item.url }}"
@ -67,84 +23,110 @@
  loop_control:
    label: "{{ item.dest }}"

+- name: Make destination directory for Nova certificates
+  file:
+    state: directory
+    path: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt/"
+  when: kolla_enable_nova | bool and kolla_libvirt_tls | bool
+
+- name: Copy client TLS certificates for Nova
+  vars:
+    certificates:
+      - clientcert.pem
+      - clientkey.pem
+      - cacert.pem
+  copy:
+    src: "{{ kolla_nova_libvirt_certificates_src }}/{{ item }}"
+    dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt/{{ item }}"
+  loop: "{{ certificates if kolla_enable_nova | bool and kolla_libvirt_tls | bool else [] }}"
+
+- name: Copy server TLS certificates for Nova
+  vars:
+    certificates:
+      - servercert.pem
+      - serverkey.pem
+  copy:
+    src: "{{ kolla_nova_libvirt_certificates_src }}/{{ item }}"
+    dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt/{{ item }}"
+  loop: "{{ certificates if kolla_enable_nova | bool and kolla_enable_nova_libvirt_container | bool and kolla_libvirt_tls | bool else [] }}"
+
 # We support a fairly flexible mechanism of dropping config file templates into
 # an 'extra' config directory, and passing these through to kolla-ansible. We
 # look for matching files in the source directory to template, and also remove
 # any unexpected files from the destination, to support removal of files.

- name: Find extra configuration files
-  find:
-    path: "{{ item.src }}"
-    patterns: "{{ item.patterns }}"
-    recurse: true
-  with_items: "{{ kolla_openstack_custom_config }}"
-  register: find_src_result
-  delegate_to: localhost
+- name: Collect details about custom config
+  kolla_custom_config_info:
+    destination: "{{ kolla_node_custom_config_path }}"
+    ignore_globs: "{{ _kolla_openstack_custom_config_cleanup_ignore_globs }}"
+    include_globs: "{{ kolla_openstack_custom_config_include_globs }}"
+    rules: "{{ kolla_openstack_custom_config_rules }}"
+    search_paths: "{{ kolla_openstack_custom_config_paths | product(['/kolla/config']) | map('join') | list }}"
+  register: kolla_custom_config_info

- name: Find previously generated extra configuration files
-  find:
-    path: "{{ item.dest }}"
-    patterns: "{{ item.patterns }}"
-  with_items: "{{ kolla_openstack_custom_config }}"
-  register: find_dest_result
+- name: Print kolla_custom_config_info when using -v
+  debug:
+    msg: "{{ kolla_custom_config_info }}"
+    verbosity: 1

 - name: Ensure extra configuration parent directories are present
  file:
-    path: "{{ item.0.item.dest }}/{{ item.1.path | relpath(item.0.item.src) | dirname }}"
+    path: "{{ item }}"
+    recurse: true
    state: directory
-    mode: 0750
-  with_subelements:
-    - "{{ find_src_result.results }}"
-    - files
-    - skip_missing: true
-  when:
-    - item.0.item.enabled | bool
-    - item.1.path | basename not in item.0.item.ignore | default([])
+  with_items: "{{ kolla_custom_config_info.create_dir }}"

- name: Ensure templated extra configuration files exist
-  template:
-    src: "{{ item.1.path }}"
-    dest: "{{ item.0.item.dest }}/{{ item.1.path | relpath(item.0.item.src) }}"
+- name: "Ensure extra configuration files exist (strategy: template)"
+  vars:
+    params:
+      src: "{{ item.src }}"
+      dest: "{{ item.dest }}"
      mode: 0640
-  with_subelements:
-    - "{{ find_src_result.results }}"
-    - files
-    - skip_missing: true
-  when:
-    - item.0.item.enabled | bool
-    - item.1.path | basename not in item.0.item.ignore | default([])
-    - item.1.path | basename not in item.0.item.untemplated | default([])
-    - (item.1.path | dirname | relpath(item.0.item.src)).split("/")[0] not in item.0.item.untemplated_dirs | default([])
+  template: "{{ params | combine(item.params) }}"
+  with_items: "{{ kolla_custom_config_info.template }}"

- name: Ensure untemplated extra configuration files exist
-  copy:
-    src: "{{ item.1.path }}"
-    dest: "{{ item.0.item.dest }}/{{ item.1.path | relpath(item.0.item.src) }}"
+- name: "Ensure extra configuration files exist (strategy: copy)"
+  vars:
+    params:
+      src: "{{ item.src }}"
+      dest: "{{ item.dest }}"
      mode: 0640
-  with_subelements:
-    - "{{ find_src_result.results }}"
-    - files
-    - skip_missing: true
-  when:
-    - item.0.item.enabled | bool
-    - item.1.path | basename not in item.0.item.ignore | default([])
-    - (item.1.path | basename in item.0.item.untemplated | default([])) or
-      ((item.1.path | dirname | relpath(item.0.item.src)).split("/")[0] in item.0.item.untemplated_dirs | default([]))
+  copy: "{{ params | combine(item.params) }}"
+  # NOTE: .copy is ambiguous with copy method
+  with_items: "{{ kolla_custom_config_info['copy'] }}"
+
+- name: "Ensure extra configuration files exist (strategy: merge_configs)"
+  vars:
+    params:
+      sources: "{{ item.sources }}"
+      dest: "{{ item.dest }}"
+      mode: 0640
+  merge_configs: "{{ params | combine(item.params) }}"
+  with_items: "{{ kolla_custom_config_info.merge_configs }}"
+
+- name: "Ensure extra configuration files exist (strategy: merge_yaml)"
+  vars:
+    params:
+      sources: "{{ item.sources }}"
+      dest: "{{ item.dest }}"
+      mode: 0640
+  merge_yaml: "{{ params | combine(item.params) }}"
+  with_items: "{{ kolla_custom_config_info.merge_yaml }}"
+
+- name: "Ensure extra configuration files exist (strategy: concat)"
+  vars:
+    params:
+      content: |
+        {%- for path in item.sources -%}
+        {{ lookup('template', path) }}
+        {%- endfor -%}
+      dest: "{{ item.dest }}"
+      mode: 0640
+  copy: "{{ params | combine(item.params) }}"
+  with_items: "{{ kolla_custom_config_info.concat }}"

 - name: Ensure unnecessary extra configuration files are absent
  file:
-    path: "{{ item.1.path }}"
+    path: "{{ item }}"
    state: absent
-  with_subelements:
-    - "{{ find_dest_result.results }}"
-    - files
-    - skip_missing: true
-  when:
-    - not item.0.item.enabled or
-      item.1.path | basename not in src_files
-    - item.1.path | basename not in item.0.item.ignore | default([])
-  vars:
-    # Find the source result that corresponds to this one.
-    src_result: "{{ (find_src_result.results | selectattr('item', 'equalto', item.0.item) | list)[0] }}"
-    # Find the list of files in the source.
-    src_files: "{{ src_result.files | map(attribute='path') | map('basename') | list }}"
+  with_items: "{{ kolla_custom_config_info.delete }}"
--- a/ansible/roles/kolla-openstack/templates/kolla/config/aodh.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/aodh.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_aodh %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/backup.my.cnf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/backup.my.cnf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_mariabackup %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/barbican.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/barbican.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_barbican %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/blazar.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/blazar.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_blazar %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/ceilometer.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/ceilometer.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_ceilometer %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/cinder.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/cinder.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_cinder %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/cloudkitty.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/cloudkitty.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_cloudkitty %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/designate.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/designate.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_designate %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/galera.cnf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/galera.cnf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_mariadb %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/glance.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/glance.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_glance %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/global.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/global.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_global %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/gnocchi.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/gnocchi.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_gnocchi %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/grafana.ini
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/grafana.ini
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_grafana %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/heat.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/heat.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_heat %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/ironic-inspector.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/ironic-inspector.conf
--- a/ansible/roles/kolla-openstack/templates/kolla/config/ironic.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/ironic.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 [DEFAULT]
 enabled_hardware_types: {{ kolla_ironic_enabled_hardware_types | join(',') }}

--- a/ansible/roles/kolla-openstack/templates/kolla/config/keystone.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/keystone.conf
@ -1,4 +1,3 @@
-# {{ ansible_managed }}
 {% if kolla_extra_keystone %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/magnum.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/magnum.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_magnum %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/manila.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/manila.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_manila %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/masakari.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/masakari.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_masakari %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/multipath.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/multipath.conf
--- a/ansible/roles/kolla-openstack/templates/kolla/config/murano.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/murano.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_murano %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/neutron.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/neutron.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_neutron %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/neutron/ml2_conf.ini
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/neutron/ml2_conf.ini
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 [ml2]
 {% if kolla_neutron_ml2_mechanism_drivers %}
 mechanism_drivers = {{ kolla_neutron_ml2_mechanism_drivers | join(',') }}
--- a/ansible/roles/kolla-openstack/templates/kolla/config/nova.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/nova.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_nova %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/octavia.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/octavia.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_octavia %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/placement.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/placement.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_placement %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/templates/kolla/config/sahara.conf
+++ b/ansible/roles/kolla-openstack/templates/kolla/config/sahara.conf
@ -1,5 +1,3 @@
-# {{ ansible_managed }}
-
 {% if kolla_extra_sahara %}
 #######################
 # Extra configuration
--- a/ansible/roles/kolla-openstack/vars/main.yml
+++ b/ansible/roles/kolla-openstack/vars/main.yml
@ -1,222 +1,16 @@
 ---
-# List of custom configuration directories.
-# Each item is a dict containing the following items:
-# src: Path to directory containing configuration file templates.
-# dest: Path to directory in which generated files will be created.
-# patterns: One or more file name patterns to match.
-# enabled: Whether these files should be templated.
-# ignore: Optional list of files to ignore. These files will not be copied to
-#         the destination, and will not be removed from the destination, even
-#         if disabled or unexpected.
-kolla_openstack_custom_config:
-  # Aodh.
-  - src: "{{ kolla_extra_config_path }}/aodh"
-    dest: "{{ kolla_node_custom_config_path }}/aodh"
-    patterns: "*"
-    enabled: "{{ kolla_enable_aodh }}"
-  # Barbican.
-  - src: "{{ kolla_extra_config_path }}/barbican"
-    dest: "{{ kolla_node_custom_config_path }}/barbican"
-    patterns: "*"
-    enabled: "{{ kolla_enable_barbican }}"
-  # Blazar.
-  - src: "{{ kolla_extra_config_path }}/blazar"
-    dest: "{{ kolla_node_custom_config_path }}/blazar"
-    patterns: "*"
-    enabled: "{{ kolla_enable_blazar }}"
-  # Ceilometer.
-  - src: "{{ kolla_extra_config_path }}/ceilometer"
-    dest: "{{ kolla_node_custom_config_path }}/ceilometer"
-    patterns: "*"
-    enabled: "{{ kolla_enable_ceilometer }}"
-  # Cinder.
-  - src: "{{ kolla_extra_config_path }}/cinder"
-    dest: "{{ kolla_node_custom_config_path }}/cinder"
-    patterns: "*"
-    enabled: "{{ kolla_enable_cinder }}"
-  # CloudKitty.
-  - src: "{{ kolla_extra_config_path }}/cloudkitty"
-    dest: "{{ kolla_node_custom_config_path }}/cloudkitty"
-    patterns: "*"
-    enabled: "{{ kolla_enable_cloudkitty }}"
-  # Designate.
-  - src: "{{ kolla_extra_config_path }}/designate"
-    dest: "{{ kolla_node_custom_config_path }}/designate"
-    patterns: "*"
-    enabled: "{{ kolla_enable_designate }}"
-  # Fluentd filters.
-  - src: "{{ kolla_extra_config_path }}//fluentd/filter"
-    dest: "{{ kolla_node_custom_config_path }}/fluentd/filter"
-    patterns: "*.conf"
-    enabled: true
-  # Fluentd inputs.
-  - src: "{{ kolla_extra_config_path }}//fluentd/input"
-    dest: "{{ kolla_node_custom_config_path }}/fluentd/input"
-    patterns: "*.conf"
-    enabled: true
-  # Fluentd outputs.
-  - src: "{{ kolla_extra_config_path }}/fluentd/output"
-    dest: "{{ kolla_node_custom_config_path }}/fluentd/output"
-    patterns: "*.conf"
-    enabled: true
-  # Glance.
-  - src: "{{ kolla_extra_config_path }}/glance"
-    dest: "{{ kolla_node_custom_config_path }}/glance"
-    patterns: "*"
-    enabled: "{{ kolla_enable_glance }}"
-  # Gnocchi.
-  - src: "{{ kolla_extra_config_path }}/gnocchi"
-    dest: "{{ kolla_node_custom_config_path }}/gnocchi"
-    patterns: "*"
-    enabled: "{{ kolla_enable_gnocchi }}"
-  # Grafana.
-  - src: "{{ kolla_extra_config_path }}/grafana"
-    dest: "{{ kolla_node_custom_config_path }}/grafana"
-    patterns: "*"
-    enabled: "{{ kolla_enable_grafana }}"
-  # HAProxy.
-  - src: "{{ kolla_extra_config_path }}/haproxy"
-    dest: "{{ kolla_node_custom_config_path }}/haproxy"
-    patterns: "*"
-    enabled: "{{ kolla_enable_haproxy }}"
-  - src: "{{ kolla_extra_config_path }}/haproxy-config"
-    dest: "{{ kolla_node_custom_config_path }}/haproxy-config"
-    patterns: "*"
-    enabled: "{{ kolla_enable_haproxy }}"
-  # Heat.
-  - src: "{{ kolla_extra_config_path }}/heat"
-    dest: "{{ kolla_node_custom_config_path }}/heat"
-    patterns: "*"
-    enabled: "{{ kolla_enable_heat }}"
-  # Horizon.
-  - src: "{{ kolla_extra_config_path }}/horizon"
-    dest: "{{ kolla_node_custom_config_path }}/horizon"
-    patterns: "*"
-    enabled: "{{ kolla_enable_horizon }}"
-    untemplated_dirs:
-    # Do not attempt to template themes directory.
-      - "themes"
-  # InfluxDB.
-  - src: "{{ kolla_extra_config_path }}/"
-    dest: "{{ kolla_node_custom_config_path }}/"
-    patterns: "influx*"
-    enabled: "{{ kolla_enable_influxdb }}"
-  # Ironic.
-  - src: "{{ kolla_extra_config_path }}/ironic"
-    dest: "{{ kolla_node_custom_config_path }}/ironic"
-    patterns: "*"
-    enabled: "{{ kolla_enable_ironic }}"
-    ignore:
-      # These are templated by kayobe, so don't remove them.
-      - ironic-agent.initramfs
-      - ironic-agent.kernel
-  # Keystone.
-  - src: "{{ kolla_extra_config_path }}/keystone"
-    dest: "{{ kolla_node_custom_config_path }}/keystone"
-    patterns: "*"
-    enabled: "{{ kolla_enable_keystone }}"
-  # Keepalived.
-  - src: "{{ kolla_extra_config_path }}/keepalived"
-    dest: "{{ kolla_node_custom_config_path }}/keepalived"
-    patterns: "*"
-    enabled: "{{ kolla_enable_haproxy }}"
-  # Magnum.
-  - src: "{{ kolla_extra_config_path }}/magnum"
-    dest: "{{ kolla_node_custom_config_path }}/magnum"
-    patterns: "*"
-    enabled: "{{ kolla_enable_magnum }}"
-  # Manila.
-  - src: "{{ kolla_extra_config_path }}/manila"
-    dest: "{{ kolla_node_custom_config_path }}/manila"
-    patterns: "*"
-    enabled: "{{ kolla_enable_manila }}"
-  # MariaDB.
-  - src: "{{ kolla_extra_config_path }}/mariadb"
-    dest: "{{ kolla_node_custom_config_path }}/mariadb"
-    patterns: "*"
-    enabled: "{{ kolla_enable_mariadb }}"
-  # Masakari.
-  - src: "{{ kolla_extra_config_path }}/masakari"
-    dest: "{{ kolla_node_custom_config_path }}/masakari"
-    patterns: "*"
-    enabled: "{{ kolla_enable_masakari }}"
-  # Murano.
-  - src: "{{ kolla_extra_config_path }}/murano"
-    dest: "{{ kolla_node_custom_config_path }}/murano"
-    patterns: "*"
-    enabled: "{{ kolla_enable_murano }}"
-  # Neutron.
-  - src: "{{ kolla_extra_config_path }}/neutron"
-    dest: "{{ kolla_node_custom_config_path }}/neutron"
-    patterns: "*"
-    enabled: "{{ kolla_enable_neutron }}"
-    ignore:
-      # These are templated by kayobe, so don't remove them.
-      - ml2_conf.ini
-  # Nova.
-  - src: "{{ kolla_extra_config_path }}/nova"
-    dest: "{{ kolla_node_custom_config_path }}/nova"
-    patterns: "*"
-    enabled: "{{ kolla_enable_nova }}"
-  - src: "{{ kolla_extra_config_path }}/nova_compute"
-    dest: "{{ kolla_node_custom_config_path }}/nova_compute"
-    patterns: "*"
-    enabled: "{{ kolla_enable_nova }}"
-  - src: "{{ kolla_nova_libvirt_certificates_src }}"
-    dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt"
-    patterns:
-      - clientcert.pem
-      - clientkey.pem
-      - cacert.pem
-    enabled: "{{ kolla_enable_nova | bool and kolla_libvirt_tls | bool }}"
-    untemplated:
-      - clientcert.pem
-      - clientkey.pem
-      - cacert.pem
-  - src: "{{ kolla_nova_libvirt_certificates_src }}"
-    dest: "{{ kolla_node_custom_config_path }}/nova/nova-libvirt"
-    patterns:
-      - servercert.pem
-      - serverkey.pem
+
+# An internal variable used for cleaning files from the generated config
+# directory. Should not be exposed as we may wish to change the cleanup
+# mechanism to something more robust.
+_kolla_openstack_custom_config_cleanup_ignore_globs:
+  - glob: ironic/ironic-agent.initramfs
+    enabled: "{{ kolla_enable_ironic | bool }}"
+  - glob: ironic/ironic-agent.kernel
+    enabled: "{{ kolla_enable_ironic | bool }}"
+  - glob: nova/nova-libvirt/server*.pem
    enabled: "{{ kolla_enable_nova | bool and kolla_enable_nova_libvirt_container | bool and kolla_libvirt_tls | bool }}"
-    untemplated:
-      - servercert.pem
-      - serverkey.pem
-  # Octavia.
-  - src: "{{ kolla_extra_config_path }}/octavia"
-    dest: "{{ kolla_node_custom_config_path }}/octavia"
-    patterns: "*"
-    enabled: "{{ kolla_enable_octavia }}"
-  # OpenSearch.
-  - src: "{{ kolla_extra_config_path }}/opensearch"
-    dest: "{{ kolla_node_custom_config_path }}/opensearch"
-    patterns: "*"
-    enabled: "{{ kolla_enable_opensearch }}"
-  # Placement
-  - src: "{{ kolla_extra_config_path }}/placement"
-    dest: "{{ kolla_node_custom_config_path }}/placement"
-    patterns: "*"
-    enabled: "{{ kolla_enable_placement }}"
-  # Prometheus config
-  - src: "{{ kolla_extra_config_path }}/prometheus"
-    dest: "{{ kolla_node_custom_config_path }}/prometheus"
-    patterns: "*"
-    enabled: "{{ kolla_enable_prometheus }}"
-  # Sahara.
-  - src: "{{ kolla_extra_config_path }}/sahara"
-    dest: "{{ kolla_node_custom_config_path }}/sahara"
-    patterns: "*"
-    enabled: "{{ kolla_enable_sahara }}"
-  # Swift.
-  - src: "{{ kolla_extra_config_path }}/swift"
-    dest: "{{ kolla_node_custom_config_path }}/swift"
-    patterns: "*"
-    enabled: "{{ kolla_enable_swift }}"
-    untemplated:
-      # These are binary files, and should not be templated.
-      - account.builder
-      - account.ring.gz
-      - container.builder
-      - container.ring.gz
-      - object.builder
-      - object.ring.gz
+  - glob: nova/nova-libvirt/client*.pem
+    enabled: "{{ kolla_enable_nova | bool and kolla_libvirt_tls | bool }}"
+  - glob: nova/nova-libvirt/cacert.pem
+    enabled: "{{ kolla_enable_nova | bool and kolla_libvirt_tls | bool }}"
--- a/doc/source/multiple-environments.rst
+++ b/doc/source/multiple-environments.rst
@ -177,10 +177,8 @@ Kolla Configuration
 -------------------

 In the Wallaby release, Kolla configuration was independent in each
-environment.
-
-As of the Xena release, the following files support combining the
-environment-specific and shared configuration file content:
+environment. The Xena release supported combining environment-specific
+and shared configuration file content for the following subset of the files:

 * ``kolla/config/bifrost/bifrost.yml``
 * ``kolla/config/bifrost/dib.yml``
@ -189,8 +187,156 @@ environment-specific and shared configuration file content:
 * ``kolla/kolla-build.conf``
 * ``kolla/repos.yml`` or ``kolla/repos.yaml``

-Options in the environment-specific files take precedence over those in the
-shared files.
+The Antelope release expands upon this list to add support for combining Kolla
+Ansible custom service configuration. This behaviour is configured using two
+variables:
+
+* ``kolla_openstack_custom_config_include_globs``: Specifies which files are
+  considered when templating the Kolla configuration. The Kayobe defaults
+  are set using ``kolla_openstack_custom_config_include_globs_default``.
+  An optional list of additional globs can be set using:
+  ``kolla_openstack_custom_config_include_globs_extra``. These are
+  combined with ``kolla_openstack_custom_config_include_globs_default``
+  to produce ``kolla_openstack_custom_config_include_globs``.
+  Each list entry is a dictionary with the following keys:
+
+   * ``enabled``: Boolean which determines if this rule is used. Set to
+     ``false`` to disable the rule.
+   * ``glob``: String glob matching a relative path in the ``kolla/config``
+     directory
+
+   An example of such a rule:
+
+   .. code-block:: yaml
+
+      enabled: '{{ kolla_enable_aodh | bool }}'
+      glob: aodh/**
+
+* ``kolla_openstack_custom_config_rules``: List of rules that specify the
+  strategy to use when generating a particular file. The Kayobe defaults
+  are set using ``kolla_openstack_custom_config_rules_default``.
+  An optional list of additional rules can be set using:
+  ``kolla_openstack_custom_config_rules_extra``. These are
+  combined with ``kolla_openstack_custom_config_rules_default``
+  to produce ``kolla_openstack_custom_config_rules``.
+  Each list entry is a dictionary with the format:
+
+   * ``glob``: A glob matching files for this rule to match on (relative to the
+     search path)
+   * ``priority``: The rules are processed in increasing priority order with the
+     first rule matching taking effect
+   * ``strategy``: How to process the matched file. One of ``copy``,
+     ``concat``, ``template``, ``merge_configs``, ``merge_yaml``
+   * ``params``: Optional list of additional params to pass to module enacting
+     the strategy
+
+   An example of such a rule:
+
+   .. code-block:: yaml
+
+      glob: a/path/test.yml
+      strategy: merge_yaml
+      priority: 1000
+      params:
+        extend_lists: true
+
+The Kayobe defaults fallback to using the ``template`` strategy, with a
+priority of 65535. To override this behaviour configure a rule with a lower
+priority e.g:
+
+   .. code-block:: yaml
+
+      glob: horizon/themes/**
+      strategy: copy
+      priority: 1000
+
+The default INI merging strategy can be configured using:
+``kolla_openstack_custom_config_ini_merge_strategy_default``. It defaults to ``concat``
+for backwards compatibility. An alternative strategy is ``merge_configs`` which will
+merge the two INI files so that values set in the environment take precedence over values
+set in the shared files. The caveat with the ``merge_configs`` strategy is that files
+must template to valid INI. This is mostly an issue when you use raw Jinja
+tags, for example:
+
+   .. code-block:: ini
+
+      [defaults]
+      {% raw %}
+      {% if inventory_hostname in 'compute' %}
+      foo=bar
+      {% else %}
+      foo=baz
+      {% endif %}
+      {% endraw %}
+
+After the first round of templating by Kayobe the raw tags are stripped. This leaves:
+
+   .. code-block:: ini
+
+      [defaults]
+      {% if inventory_hostname in 'compute' %}
+      foo=bar
+      {% else %}
+      foo=baz
+      {% endif %}
+
+Which isn't valid INI (due to the Jinja if blocks) and cannot be merged. In most cases
+the templating can be refactored:
+
+   .. code-block:: ini
+
+      [defaults]
+      {% raw %}
+      foo={{ 'bar' if inventory_hostname in 'compute' else 'baz' }}
+      {% endraw %}
+
+Alternatively, you can use Kolla host or group variables.
+
+Disabling the default rules
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are some convenience variables to disable a subset of the
+rules in ``kolla_openstack_custom_config_rules_default``:
+
+* ``kolla_openstack_custom_config_rules_default_remove``: Allows you remove
+  a rule by matching on the glob:
+
+   .. code-block:: yaml
+
+      kolla_openstack_custom_config_rules_default_remove:
+         - "**/*.ini"
+
+* ``kolla_openstack_custom_config_merge_configs_enabled``: Enables rules for
+  matching INI files. Default is ``true``.
+
+* ``kolla_openstack_custom_config_merge_yaml_enabled``: Enables rules for
+  matching YAML files. Default is ``true``.
+
+These allow you to more easily keep in sync with the upstream defaults. If
+you had an override on ``kolla_openstack_custom_config_rules``, that
+replicated most of ``kolla_openstack_custom_config_rules_default`` you'd have
+to keep this in sync with the upstream kayobe defaults.
+
+Search paths
+^^^^^^^^^^^^
+
+When merging config files the following locations are "searched" to find
+files with an identical relative path:
+
+- ``<environment-path>/kolla/config``
+- ``<shared-files-path>/kolla/config``
+- ``<kolla-openstack-role-path>/templates/kolla/config``
+
+Not all strategies use all of the files when generating the kolla config.
+For instance, the copy strategy will use the first file found when searching
+each of the paths.
+
+There is a feature flag: ``kolla_openstack_custom_config_environment_merging_enabled``,
+that may be set to ``false`` to prevent Kayobe searching the shared files path
+when merging configs. This is to replicate the legacy behaviour where the
+environment Kolla custom service configuration was not merged with the base
+layer. We still merge the files with Kayobe's defaults in the
+``kolla-openstack`` role's internal templates.

 Managing Independent Environment Files
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@ -447,6 +447,79 @@
 #kolla_enable_watcher:
 #kolla_enable_zun:

+###############################################################################
+# Kolla custom config generation.
+
+# Feature flag to add $KAYOBE_CONFIG_PATH to the list of search paths used
+# when searching for Kolla custom service configuration. Only has an effect in
+# a multiple environments setup. This allows you to configure merging between
+# your environment and the base layer. Defaults to true. Set to false to for
+# backwards compatability.
+#kolla_openstack_custom_config_environment_merging_enabled:
+
+# Default value for kolla_openstack_custom_config_include_globs.
+#kolla_openstack_custom_config_include_globs_default:
+
+# Extra items to add to kolla_openstack_custom_config_include_globs_default
+# to produce kolla_openstack_custom_config_include_globs.
+#kolla_openstack_custom_config_include_globs_extra:
+
+# List of dictionaries with the following keys:
+#   glob: a glob pattern. Any files matching this pattern will be copied to the
+#         the kolla custom config directory
+#   enabled: boolean to disable the glob.
+# This determines the list of files to copy to the generated kolla config
+# directory.
+#kolla_openstack_custom_config_include_globs:
+
+# Kolla config generation rules. These operate on the list of files produced by
+# applying kolla_openstack_custom_config_include_globs. Each of the paths in
+# kolla_openstack_custom_config_paths is searched for files matching one of the
+# globs. If a match is found, any files with the same relative path are grouped
+# together. The rules determine what to do with these matching files e.g copy
+# the most specific file without templating, merge the files with
+# merge_configs, etc.
+# List of dictionaries with the following keys:
+#   glob: A glob matching files for this rule to match on (relative to the
+#     search path)
+#   priority: The rules are processed in increasing priority order with the
+#     first rule matching taking effect.
+#   strategy: How to process the matched file. One of copy, concat, template,
+#      merge_configs, merge_yaml
+#   params: List of params to pass to module enacting the strategy
+# Strategies:
+#   copy: Copy most specific file to kolla config without templating
+#   template: Template most specific file to kolla config
+#   concat: Concatenate files and copy the result to generated kolla config
+#   merge_configs: Use the merge_configs module to merge an ini file, before
+#     copying to the generated kolla-config.
+#   merge_yaml: Use the merge_yaml module to merge a file, before copying to
+#     the generated kolla-config.
+#kolla_openstack_custom_config_rules:
+
+# Whether to enable ini merging rules in
+# kolla_openstack_custom_config_rules_default. Default is true.
+#kolla_openstack_custom_config_merge_configs_enabled:
+
+# Whether to enable yaml merging rules in
+# kolla_openstack_custom_config_rules_default. Default is true.
+#kolla_openstack_custom_config_merge_yaml_enabled:
+
+# Default merge strategy for ini files in
+# kolla_openstack_custom_config_rules_default. Default is concat.
+#kolla_openstack_custom_config_ini_merge_strategy_default:
+
+# Default value for kolla_openstack_custom_config_rules.
+#kolla_openstack_custom_config_rules_default:
+
+# List of globs to filter from kolla_openstack_custom_config_rules_default.
+# Default is an empty list.
+#kolla_openstack_custom_config_rules_default_remove:
+
+# Extra items to add to kolla_openstack_custom_config_rules_default
+# to produce kolla_openstack_custom_config_rules.
+#kolla_openstack_custom_config_rules_extra:
+
 ###############################################################################
 # Passwords and credentials.

--- a/kayobe/tests/molecule/utils.py
+++ b/kayobe/tests/molecule/utils.py
@ -12,6 +12,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.

+import re
+
 import configparser
 from io import StringIO

@ -53,6 +55,21 @@ def test_ini_file(host, path, owner='root', group='root', expected=None):
            assert parser.get(exp_section_name, exp_key) == exp_value


+def test_regex_in_file(host, path, owner='root', group='root', regex=None):
+    """Test that a regex exists in file
+
+    Validate that the file exists, has the correct ownership, format and
+    expected contents.
+
+    :param regex to search for in file
+    """
+    test_file(host, path, owner, group)
+
+    matches = re.findall(regex, host.file(path).content_string)
+
+    assert len(matches) > 0
+
+
 def test_directory(host, path, owner='root', group='root'):
    """Test an expected directory.

--- a/releasenotes/notes/feature-merge-kolla-configs-99773e2f0af2ea4b.yaml
+++ b/releasenotes/notes/feature-merge-kolla-configs-99773e2f0af2ea4b.yaml
@ -0,0 +1,52 @@
+---
+features:
+  - |
+    Adds new functionality to merge Kolla custom service configuration in a
+    Kayobe environment with Kolla configuration in the base configuration
+    layer.
+upgrade:
+  - |
+    Environment-specific Kolla custom service configuration is now merged with
+    Kolla configuration in the base configuration layer.  Config options
+    duplicated in the base layer and the environment will need to be
+    de-deduplicated to avoid the config option showing up multiple times in the
+    generated output (although in general this should not be a problem).
+
+    Set ``kolla_openstack_custom_config_environment_merging_enabled`` to
+    ``false`` to revert back to the previous behavior where only the config in
+    the environment was considered.
+deprecations:
+  - |
+    Deprecates the following variables for removal in the Bobcat release:
+
+    * ``kolla_extra_global``
+    * ``kolla_extra_aodh``
+    * ``kolla_extra_barbican``
+    * ``kolla_extra_blazar``
+    * ``kolla_extra_ceilometer``
+    * ``kolla_extra_cinder``
+    * ``kolla_extra_cloudkitty``
+    * ``kolla_extra_designate``
+    * ``kolla_extra_gnocchi``
+    * ``kolla_extra_grafana``
+    * ``kolla_extra_heat``
+    * ``kolla_extra_ironic``
+    * ``kolla_extra_inspector``
+    * ``kolla_extra_keystone``
+    * ``kolla_extra_magnum``
+    * ``kolla_extra_mariabackup``
+    * ``kolla_extra_mariadb``
+    * ``kolla_extra_manila``
+    * ``kolla_extra_masakari``
+    * ``kolla_extra_murano``
+    * ``kolla_extra_neutron``
+    * ``kolla_extra_neutron_ml2``
+    * ``kolla_extra_nova``
+    * ``kolla_extra_octavia``
+    * ``kolla_extra_placement``
+    * ``kolla_extra_sahara``
+
+    Use of Kolla custom service configuration files in
+    ``etc/kayobe/kolla/config`` and
+    ``etc/kayobe/environments/<environment>/kolla/config`` should be used
+    instead.
--- a/requirements.txt
+++ b/requirements.txt
@ -9,4 +9,5 @@ selinux # MIT
 oslo.config>=5.2.0 # Apache-2.0
 paramiko # LGPL
 jsonschema<5 # MIT
+wcmatch>=8.2,<=9.0 # MIT
 hvac>=0.10.1
--- a/tools/test-molecule.sh
+++ b/tools/test-molecule.sh
@ -7,6 +7,12 @@ set -e

 molecules="$(find ansible/roles/ -name molecule -type d)"

+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# FIXME: doesn't get passed through to linter.
+export ANSIBLE_ACTION_PLUGINS="$PARENT/../kayobe/plugins/action:~/.ansible/plugins/action:/usr/share/ansible/plugins/action"
+export ANSIBLE_FORCE_COLOR=True
+
 failed=0
 ran=0
 for molecule in $molecules; do