Performance: use a single config file for fluentd

Currently we generate multiple fluentd configuration files for inputs,
filters, formatters and outputs.
These are then included from the main td-agent.conf configuration file.
With a large number of hosts, this can take a long time to template.

Benchmarking of templating is available at [1].

This change switches to a single fluentd configuration file, with the
include done locally. For the default template files included with Kolla
Ansible we use Jinja includes, but this does not work with templates in
a different directory. We therefore use the Ansible template lookup
plugin, which has a slightly higher overhead than a jinja include, but
far lower than generating multiple templates. This should drastically
improve the performance of this task.

[1] https://github.com/stackhpc/ansible-scaling/blob/master/doc/template.md

Partially-Implements: blueprint performance-improvements

Change-Id: Ia8623be0aa861fea3e54d2c9e1c971dfd8e3afa9

ansible/roles/common/filter_plugins/filters.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2020 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from kolla_ansible import fluentd_filters
+
+
+class FilterModule(object):
+    """Service filters."""
+
+    def filters(self):
+        return fluentd_filters.get_filters()

ansible/roles/common/tasks/config.yml

@@ -18,10 +18,6 @@
       - service_name: "fluentd"
         paths:
           - "fluentd"
-          - "fluentd/input"
-          - "fluentd/output"
-          - "fluentd/format"
-          - "fluentd/filter"
       - service_name: "kolla-toolbox"
         paths:
           - "kolla-toolbox"
@@ -78,46 +74,22 @@
   delegate_to: localhost
   when: common_services.fluentd | service_enabled_and_mapped_to_host
 
-- name: Copying over fluentd input config files
+- name: Find custom fluentd filter config files
-  vars:
+  find:
-    customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | map('basename') | list }}"
+    path: "{{ node_custom_config }}/fluentd/filter"
-  template:
+    pattern: "*.conf"
-    src: "conf/input/{{ item }}.conf.j2"
+  run_once: True
-    dest: "{{ node_config_directory }}/fluentd/input/{{ item }}.conf"
+  register: find_custom_fluentd_filters
-    mode: "0660"
+  delegate_to: localhost
-  become: true
+  when: common_services.fluentd | service_enabled_and_mapped_to_host
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-    - item ~ '.conf' not in customised_input_files
-  with_items:
-    - "00-global"
-    - "01-syslog"
-    - "02-mariadb"
-    - "03-rabbitmq"
-    - "04-openstack-wsgi"
-    - "05-libvirt"
-    - "06-zookeeper"
-    - "07-kafka"
-    - "09-monasca"
-  notify:
-    - Restart fluentd container
 
-- name: Copying over custom fluentd input config files
+- name: Find custom fluentd format config files
-  template:
+  find:
-    src: "{{ item.path }}"
+    path: "{{ node_custom_config }}/fluentd/format"
-    dest: "{{ node_config_directory }}/fluentd/input/{{ item.path | basename }}"
+    pattern: "*.conf"
-    mode: "0660"
+  run_once: True
-  when:
+  register: find_custom_fluentd_formats
-    - common_services.fluentd | service_enabled_and_mapped_to_host
+  delegate_to: localhost
-  with_items: "{{ find_custom_fluentd_inputs.files }}"
-  notify:
-    - Restart fluentd container
-
-- name: Determine whether logs should be forwarded directly to Elasticsearch
-  set_fact:
-    log_direct_to_elasticsearch: "{{ ( enable_elasticsearch | bool or
-                ( elasticsearch_address != kolla_internal_vip_address )) and
-                not enable_monasca | bool }}"
   when:
     - common_services.fluentd | service_enabled_and_mapped_to_host
 
@@ -131,146 +103,57 @@
   when:
     - common_services.fluentd | service_enabled_and_mapped_to_host
 
-- name: Copying over fluentd output config files
-  vars:
-    customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | map('basename') | list }}"
-  template:
-    src: "conf/output/{{ item.name }}.conf.j2"
-    dest: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf"
-    mode: "0660"
-  become: true
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-    - item.enabled | bool
-    - item.name ~ '.conf' not in customised_output_files
-  with_items:
-    - name: "00-local"
-      enabled: true
-    - name: "01-es"
-      enabled: "{{ log_direct_to_elasticsearch }}"
-    - name: "02-monasca"
-      enabled: "{{ enable_monasca | bool }}"
-  notify:
-    - Restart fluentd container
-
-- name: Removing stale output config files
-  file:
-    path: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf"
-    state: "absent"
-  become: true
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-    - item.disable | bool
-  with_items:
-    - name: "02-monasca"
-      disable: "{{ not enable_monasca | bool }}"
-    - name: "01-es"
-      disable: "{{ not log_direct_to_elasticsearch }}"
-  notify:
-    - Restart fluentd container
-
-- name: Copying over custom fluentd output config files
-  template:
-    src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/fluentd/output/{{ item.path | basename }}"
-    mode: "0660"
-  become: true
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-  with_items: "{{ find_custom_fluentd_outputs.files }}"
-  notify:
-    - Restart fluentd container
-
-- name: Find custom fluentd format config files
-  find:
-    path: "{{ node_custom_config }}/fluentd/format"
-    pattern: "*.conf"
-  run_once: True
-  register: find_custom_fluentd_format
-  delegate_to: localhost
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-
-- name: Copying over fluentd format config files
-  vars:
-    customised_format_files: "{{ find_custom_fluentd_format.files | map(attribute='path') | map('basename') | list }}"
-  template:
-    src: "conf/format/{{ item }}.conf.j2"
-    dest: "{{ node_config_directory }}/fluentd/format/{{ item }}.conf"
-    mode: "0660"
-  become: true
-  with_items:
-    - "apache_access"
-    - "wsgi_access"
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-    - item ~ '.conf' not in customised_format_files
-  notify:
-    - Restart fluentd container
-
-- name: Copying over custom fluentd format config files
-  template:
-    src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/fluentd/format/{{ item.path | basename }}"
-    mode: "0660"
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-  with_items: "{{ find_custom_fluentd_format.files }}"
-  notify:
-    - Restart fluentd container
-
-- name: Find custom fluentd filter config files
-  find:
-    path: "{{ node_custom_config }}/fluentd/filter"
-    pattern: "*.conf"
-  run_once: True
-  register: find_custom_fluentd_filters
-  delegate_to: localhost
-  when: common_services.fluentd | service_enabled_and_mapped_to_host
-
-- name: Copying over fluentd filter config files
-  vars:
-    customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}"
-    fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
-  template:
-    src: "conf/filter/{{ item.src }}.conf.j2"
-    dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf"
-    mode: "0660"
-  become: true
-  with_items:
-    - src: 00-record_transformer
-      dest: 00-record_transformer
-    - src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}"
-      dest: 01-rewrite
-    - src: 02-parser
-      dest: 02-parser
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-    - item.src ~ '.conf' not in customised_filter_files
-  notify:
-    - Restart fluentd container
-
-- name: Copying over custom fluentd filter config files
-  template:
-    src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/fluentd/filter/{{ item.path | basename }}"
-    mode: "0660"
-  become: true
-  with_items: "{{ find_custom_fluentd_filters.files }}"
-  when:
-    - common_services.fluentd | service_enabled_and_mapped_to_host
-  notify:
-    - Restart fluentd container
-
 - name: Copying over td-agent.conf
+  vars:
+    log_direct_to_elasticsearch: >-
+      {{ ( enable_elasticsearch | bool or
+           ( elasticsearch_address != kolla_internal_vip_address )) and
+         not enable_monasca | bool }}
+    fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
+    # Inputs
+    fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}"
+    default_input_files:
+      - "conf/input/00-global.conf.j2"
+      - "conf/input/01-syslog.conf.j2"
+      - "conf/input/02-mariadb.conf.j2"
+      - "conf/input/03-rabbitmq.conf.j2"
+      - "conf/input/04-openstack-wsgi.conf.j2"
+      - "conf/input/05-libvirt.conf.j2"
+      - "conf/input/06-zookeeper.conf.j2"
+      - "conf/input/07-kafka.conf.j2"
+      - "conf/input/09-monasca.conf.j2"
+    customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}"
+    # Filters
+    fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}"
+    default_filter_files:
+      - "conf/filter/00-record_transformer.conf.j2"
+      - "conf/filter/{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}.conf.j2"
+      - "conf/filter/02-parser.conf.j2"
+    customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | list }}"
+    # Formats
+    fluentd_format_files: "{{ default_format_files | customise_fluentd(customised_format_files) }}"
+    default_format_files:
+      - "conf/format/apache_access.conf.j2"
+      - "conf/format/wsgi_access.conf.j2"
+    customised_format_files: "{{ find_custom_fluentd_formats.files | map(attribute='path') | list }}"
+    # Outputs
+    fluentd_output_files: "{{ default_output_files_enabled | customise_fluentd(customised_output_files) }}"
+    default_output_files_enabled: "{{ default_output_files | selectattr('enabled') | map(attribute='name') | list }}"
+    default_output_files:
+      - name: "conf/output/00-local.conf.j2"
+        enabled: true
+      - name: "conf/output/01-es.conf.j2"
+        enabled: "{{ log_direct_to_elasticsearch }}"
+      - name: "conf/output/02-monasca.conf.j2"
+        enabled: "{{ enable_monasca | bool }}"
+    customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | list }}"
   template:
     src: "td-agent.conf.j2"
-    dest: "{{ node_config_directory }}/{{ item }}/td-agent.conf"
+    dest: "{{ node_config_directory }}/fluentd/td-agent.conf"
     mode: "0660"
   become: true
-  with_items:
+  when:
-    - "fluentd"
+    - common_services.fluentd | service_enabled_and_mapped_to_host
-  when: common_services.fluentd | service_enabled_and_mapped_to_host
   notify:
     - Restart fluentd container
 

ansible/roles/common/templates/fluentd.json.j2

@@ -1,15 +1,6 @@
 {% set fluentd_user = fluentd_binary %}
 {% set fluentd_dir = '/etc/' ~ fluentd_binary %}
 
-{%- macro config_directory(dir) -%}
-        {
-            "source": "{{ container_config_directory }}/{{ dir }}",
-            "dest": "{{ fluentd_dir }}/{{ dir }}",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        }
-{%- endmacro -%}
-
 {% if fluentd_binary == 'fluentd' %}
     {% set fluentd_conf = 'fluent.conf' %}
     {% if kolla_base_distro in ['ubuntu', 'debian'] %}
@@ -22,14 +13,6 @@
     {% set fluentd_cmd = '/usr/sbin/td-agent' %}
 {% endif %}
 
-{%- macro config_directory_permissions(dir) -%}
-        {
-            "path": "{{ fluentd_dir }}/{{ dir }}",
-            "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
-            "perm": "0700"
-        }
-{%- endmacro -%}
-
 {
     "command": "{{ fluentd_cmd }} -o /var/log/kolla/fluentd/fluentd.log",
     "config_files": [
@@ -38,12 +21,7 @@
             "dest": "{{ fluentd_dir }}/{{ fluentd_conf }}",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
-        },
+        }
-        {# Copy all files in the following directories #}
-        {{ config_directory("input") }},
-        {{ config_directory("filter") }},
-        {{ config_directory("format") }},
-        {{ config_directory("output") }}
     ],
     "permissions": [
         {
@@ -65,12 +43,7 @@
             "path": "/var/lib/fluentd/data",
             "owner": "{{ fluentd_user }}:{{ fluentd_user }}",
             "recurse": true
-        },
+        }
-        {# Allow Fluentd to read configuration from folders #}
-        {{ config_directory_permissions("input") }},
-        {{ config_directory_permissions("filter") }},
-        {{ config_directory_permissions("format") }},
-        {{ config_directory_permissions("output") }}
     ]
 
 }

ansible/roles/common/templates/td-agent.conf.j2

@@ -1,4 +1,45 @@
-@include input/*.conf
+#jinja2: trim_blocks: False
-@include filter/*.conf
+{# Ansible restricts Jinja includes to the same directory or subdirectory of a
-@include format/*.conf
+   template. To support customised configuration outside of this path we use
-@include output/*.conf
+   the template lookup plugin. Jinja includes have a lower overhead, so we use
+   those where possible. #}
+
+# Inputs
+{%- for path in fluentd_input_files %}
+# Included from {{ path }}:
+{%- if path.startswith('/') %}
+{{ lookup('template', path) }}
+{%- else %}
+{% include path %}
+{%- endif %}
+{%- endfor %}
+
+# Filters
+{%- for path in fluentd_filter_files %}
+# Included from {{ path }}:
+{%- if path.startswith('/') %}
+{{ lookup('template', path) }}
+{%- else %}
+{% include path %}
+{%- endif %}
+{%- endfor %}
+
+# Formats
+{%- for path in fluentd_format_files %}
+# Included from {{ path }}:
+{%- if path.startswith('/') %}
+{{ lookup('template', path) }}
+{%- else %}
+{% include path %}
+{%- endif %}
+{%- endfor %}
+
+# Outputs
+{%- for path in fluentd_output_files %}
+# Included from {{ path }}:
+{%- if path.startswith('/') %}
+{{ lookup('template', path) }}
+{%- else %}
+{% include path %}
+{%- endif %}
+{%- endfor %}

kolla_ansible/fluentd_filters.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2020 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os.path
+
+
+def customise_fluentd(default_paths, customised_paths):
+    """Return a sorted list of templates for fluentd.
+
+    :param default_paths: Iterable of default template paths.
+    :param customised_paths: Iterable of customised template paths.
+    :returns: A sorted combined list of template paths.
+    """
+
+    def _basename_no_ext(path):
+        """Return the basename of a path, stripping off any extension."""
+        return os.path.splitext(os.path.basename(path))[0]
+
+    customised_file_names = {os.path.basename(f) for f in customised_paths}
+    # Starting with the default paths, remove any that have been overridden,
+    # ignoring the .j2 extension of default paths.
+    result = {f for f in default_paths
+              if _basename_no_ext(f) not in customised_file_names}
+    # Add all customised paths.
+    result.update(customised_paths)
+    # Sort by the basename of the paths.
+    return sorted(result, key=os.path.basename)
+
+
+def get_filters():
+    return {
+        "customise_fluentd": customise_fluentd,
+    }

kolla_ansible/tests/unit/test_fluentd_filters.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2020 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from kolla_ansible.fluentd_filters import customise_fluentd
+
+
+class TestFilters(unittest.TestCase):
+
+    def test_customise_fluentd_no_files(self):
+        default_files = [
+        ]
+        customised_files = [
+        ]
+        expected = [
+        ]
+        result = customise_fluentd(default_files, customised_files)
+        self.assertEqual(expected, result)
+
+    def test_customise_fluentd_no_customised_files(self):
+        default_files = [
+            "foo/bar.conf.j2"
+        ]
+        customised_files = [
+        ]
+        expected = [
+            "foo/bar.conf.j2"
+        ]
+        result = customise_fluentd(default_files, customised_files)
+        self.assertEqual(expected, result)
+
+    def test_customise_fluentd_no_default_files(self):
+        default_files = [
+        ]
+        customised_files = [
+            "foo/bar.conf"
+        ]
+        expected = [
+            "foo/bar.conf"
+        ]
+        result = customise_fluentd(default_files, customised_files)
+        self.assertEqual(expected, result)
+
+    def test_customise_fluentd_both(self):
+        default_files = [
+            "foo/bar.conf.j2"
+        ]
+        customised_files = [
+            "baz/qux.conf"
+        ]
+        expected = [
+            "foo/bar.conf.j2",
+            "baz/qux.conf"
+        ]
+        result = customise_fluentd(default_files, customised_files)
+        self.assertEqual(expected, result)
+
+    def test_customise_fluentd_override(self):
+        default_files = [
+            "foo/bar.conf.j2"
+        ]
+        customised_files = [
+            "baz/bar.conf"
+        ]
+        expected = [
+            "baz/bar.conf"
+        ]
+        result = customise_fluentd(default_files, customised_files)
+        self.assertEqual(expected, result)
+
+    def test_customise_fluentd_both_with_override(self):
+        default_files = [
+            "foo/bar.conf.j2",
+            "baz/qux.conf.j2"
+        ]
+        customised_files = [
+            "baz/bar.conf"
+        ]
+        expected = [
+            "baz/bar.conf",
+            "baz/qux.conf.j2"
+        ]
+        result = customise_fluentd(default_files, customised_files)
+        self.assertEqual(expected, result)

releasenotes/notes/fluentd-single-config-d5ae95fecbfb6e3e.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Improves performance of the ``common`` role by generating all fluentd
+    configuration in a single file.