From 1b4430314cea3bded83875e613668a0eeec5bbe1 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 6 Mar 2018 17:29:27 +0100 Subject: [PATCH] Allow custom fluentd input configurations Co-authored-by: Mark Goddard Change-Id: I2e5ecf5b01cc842ec480fc4d883a7d2283fc1c31 --- ansible/roles/common/handlers/main.yml | 1 + ansible/roles/common/tasks/config.yml | 21 +++++++++++ .../roles/common/templates/fluentd.json.j2 | 36 +++---------------- .../reference/central-logging-guide.rst | 22 ++++++++++-- ...-custom-input-config-db2675a1356717e6.yaml | 7 ++++ 5 files changed, 53 insertions(+), 34 deletions(-) create mode 100644 releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml index 96a36d1926..0887c18b92 100644 --- a/ansible/roles/common/handlers/main.yml +++ b/ansible/roles/common/handlers/main.yml @@ -19,6 +19,7 @@ - service.enabled | bool - config_json.changed | bool or fluentd_input.changed | bool + or fluentd_input_custom.changed | bool or fluentd_output.changed | bool or fluentd_output_custom.changed | bool or fluentd_format.changed | bool diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index cc663e2755..e9934164df 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -58,6 +58,27 @@ notify: - Restart fluentd container +- name: Find custom fluentd input config files + local_action: + module: find + path: "{{ node_custom_config }}/fluentd/input" + pattern: "*.conf" + run_once: True + register: find_custom_fluentd_inputs + when: + - enable_fluentd | bool + +- name: Copying over custom fluentd input config files + template: + src: "{{ item.path }}" + dest: "{{ node_config_directory }}/fluentd/input/{{ item.path | basename }}" + register: fluentd_input_custom + when: + - enable_fluentd | bool + with_items: "{{ find_custom_fluentd_inputs.files }}" + notify: + - Restart fluentd container + - name: Copying over fluentd output config files template: src: "conf/output/{{ item.name }}.conf.j2" diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2 index 55a3f37c0d..b8d16f47da 100644 --- a/ansible/roles/common/templates/fluentd.json.j2 +++ b/ansible/roles/common/templates/fluentd.json.j2 @@ -11,39 +11,11 @@ "owner": "{{ fluentd_user }}", "perm": "0600" }, + {# Copy all configuration files in input/ directory to include #} + {# custom input configs. #} { - "source": "{{ container_config_directory }}/input/00-global.conf", - "dest": "{{ fluentd_dir }}/input/00-global.conf", - "owner": "{{ fluentd_user }}", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/input/01-syslog.conf", - "dest": "{{ fluentd_dir }}/input/01-syslog.conf", - "owner": "{{ fluentd_user }}", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/input/02-mariadb.conf", - "dest": "{{ fluentd_dir }}/input/02-mariadb.conf", - "owner": "{{ fluentd_user }}", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/input/03-rabbitmq.conf", - "dest": "{{ fluentd_dir }}/input/03-rabbitmq.conf", - "owner": "{{ fluentd_user }}", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/input/04-openstack-wsgi.conf", - "dest": "{{ fluentd_dir }}/input/04-openstack-wsgi.conf", - "owner": "{{ fluentd_user }}", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/input/05-libvirt.conf", - "dest": "{{ fluentd_dir }}/input/05-libvirt.conf", + "source": "{{ container_config_directory }}/input/*.conf", + "dest": "{{ fluentd_dir }}/input/", "owner": "{{ fluentd_user }}", "perm": "0600" }, diff --git a/doc/source/reference/central-logging-guide.rst b/doc/source/reference/central-logging-guide.rst index 4ec165817c..540cd63436 100644 --- a/doc/source/reference/central-logging-guide.rst +++ b/doc/source/reference/central-logging-guide.rst @@ -164,8 +164,16 @@ can also be exported at once by choosing *export everything* option. In the same tab (Settings - Objects) one can also import saved items by choosing *import* option. +Custom log rules +~~~~~~~~~~~~~~~~ + +Kolla-Ansible automatically deploys Fluentd for forwarding OpenStack logs +from across the control plane to a central logging repository. The Fluentd +configuration is split into four parts: Input, forwarding, filtering and +formatting. The following can be customised: + Custom log filtering -==================== +-------------------- In some scenarios it may be useful to apply custom filters to logs before forwarding them. This may be useful to add additional tags to the messages @@ -177,7 +185,7 @@ configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the control host. Custom log forwarding -~~~~~~~~~~~~~~~~~~~~~ +--------------------- In some scenarios it may be useful to forward logs to a logging service other than elasticsearch. This can be done by configuring custom fluentd outputs. @@ -185,3 +193,13 @@ than elasticsearch. This can be done by configuring custom fluentd outputs. Configuration of custom fluentd outputs is possible by placing output configuration files in ``/etc/kolla/config/fluentd/output/*.conf`` on the control host. + +Custom log inputs +----------------- + +In some scenarios it may be useful to input logs from other services, e.g. +network equipment. This can be done by configuring custom fluentd inputs. + +Configuration of custom fluentd inputs is possible by placing input +configuration files in ``/etc/kolla/config/fluentd/input/*.conf`` on the +control host. diff --git a/releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml b/releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml new file mode 100644 index 0000000000..3323e84463 --- /dev/null +++ b/releasenotes/notes/fluentd-custom-input-config-db2675a1356717e6.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds ability to configure custom fluentd inputs. + + Configuration of custom fluentd inputs is possible by placing + input configuration files in /etc/kolla/config/fluentd/input/\*.conf.