diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 2da793ba33..056ecda10f 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -236,6 +236,8 @@ dest: 00-record_transformer - src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}" dest: 01-rewrite + - src: 02-parser + dest: 02-parser when: - enable_fluentd | bool - item.src ~ '.conf' not in customised_filter_files diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 index edb201d928..c8c821b3a0 100644 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 @@ -37,3 +37,10 @@ rewriterule34 programname ^(monasca-api|monasca-notification|monasca-persister|agent-collector|agent-forwarder|agent-statsd)$ openstack_python rewriterule35 programname .+ unmatched + +# Retag log messages from MariaDB according to log format + + @type rewrite_tag_filter + rewriterule1 Payload ^\d{6} infra.mariadb.mysqld_safe + rewriterule2 Payload ^\d{4}-\d{2}-\d{2} infra.mariadb.mysqld + diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 index a85735bdbc..8cafb7a721 100644 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 @@ -187,3 +187,18 @@ tag unmatched + +# Retag log messages from MariaDB according to log format + + @type rewrite_tag_filter + + key Payload + pattern /^\d{6}/ + tag infra.mariadb.mysqld_safe + + + key Payload + pattern /^\d{4}-\d{2}-\d{2}/ + tag infra.mariadb.mysqld + + diff --git a/ansible/roles/common/templates/conf/filter/02-parser.conf.j2 b/ansible/roles/common/templates/conf/filter/02-parser.conf.j2 new file mode 100644 index 0000000000..5936a3cf54 --- /dev/null +++ b/ansible/roles/common/templates/conf/filter/02-parser.conf.j2 @@ -0,0 +1,27 @@ +# Parse MariaDB logs with 6 digit date format (mysqld_safe) + + @type parser + format /^(?\d{6} {1,2}\d{1,2}:\d{1,2}:\d{1,2}) +(?mysqld_safe .*)/ + time_format %y%m%d %k:%M:%S + time_key Timestamp + key_name Payload + reserve_data true + + +# Parse MariaDB logs with 8 digit date format (mysqld) + + @type parser + format /^(?\d{4}-\d{2}-\d{2} {1,2}\d{1,2}:\d{1,2}:\d{1,2}) +(?\w+ +(\[(?\w+)\]|\w+: +(?\w+):).*)/ + time_format %Y-%m-%d %k:%M:%S + time_key Timestamp + key_name Payload + reserve_data true + + +# Re-add timestamp record now that the log date has been parsed + + @type record_transformer + + timestamp ${time} + + diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 index 3933a38568..98797b09b8 100644 --- a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 +++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 @@ -1,11 +1,12 @@ +# mysqld and its wrapper script mysqld_safe output logs using a different timestamp. +# Defer parsing the logs until the different formats have been retagged. @type tail path /var/log/kolla/mariadb/mariadb.log pos_file /var/run/{{ fluentd_binary }}/mariadb.pos tag infra.mariadb format multiline - format_firstline /^\d{6}/ - format1 /^(?