From 1ae10375f70a3b12b080d3713ae8df00f3a83e14 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Fri, 8 Jun 2018 10:10:56 +0100 Subject: [PATCH] Support deploying Monasca Log Metrics The log metrics service generates metrics from log messages which allows further analysis and alerting to be performed on them. Basic configuration is provided so that metrics are generated for high level warning logs such as error, or warning. Change-Id: I45cc17817c716296451f620f304c0b1108162a56 Partially-Implements: blueprint monasca-roles --- ansible/inventory/all-in-one | 3 + ansible/inventory/multinode | 3 + ansible/roles/monasca/defaults/main.yml | 11 +++ ansible/roles/monasca/handlers/main.yml | 21 +++++ ansible/roles/monasca/tasks/config.yml | 18 +++++ ansible/roles/monasca/tasks/deploy.yml | 6 +- .../monasca-log-metrics/log-metrics.conf.j2 | 76 +++++++++++++++++++ .../monasca-log-metrics.json.j2 | 18 +++++ ...-monasca-log-metrics-370846df015ff96a.yaml | 5 ++ 9 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 create mode 100644 ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 create mode 100644 releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index c6907097ab..25dd8dc1a4 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -455,6 +455,9 @@ monasca [monasca-log-persister:children] monasca +[monasca-log-metrics:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index b796c2986a..c64e74cdab 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -464,6 +464,9 @@ monasca [monasca-log-persister:children] monasca +[monasca-log-metrics:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 55c2f125dc..309d545dd3 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -40,6 +40,16 @@ monasca_services: - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" dimensions: "{{ monasca_log_persister_dimensions }}" + monasca-log-metrics: + container_name: monasca_log_metrics + group: monasca-log-metrics + enabled: true + image: "{{ monasca_logstash_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-log-metrics/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" + dimensions: "{{ monasca_log_metrics_dimensions }}" #################### # Databases @@ -94,6 +104,7 @@ monasca_api_dimensions: "{{ default_container_dimensions }}" monasca_log_api_dimensions: "{{ default_container_dimensions }}" monasca_log_transformer_dimensions: "{{ default_container_dimensions }}" monasca_log_persister_dimensions: "{{ default_container_dimensions }}" +monasca_log_metrics_dimensions: "{{ default_container_dimensions }}" #################### diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index d20d442741..e47955b1e0 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -87,3 +87,24 @@ or monasca_log_persister_confs.changed | bool or monasca_log_persister_elasticsearch_template.changed | bool or monasca_log_persister_container.changed | bool + +- name: Restart monasca-log-metrics container + vars: + service_name: "monasca-log-metrics" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_log_metrics_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_log_metrics_confs.changed | bool + or monasca_log_metrics_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 454452270e..4c07bd31b7 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -156,6 +156,24 @@ notify: - Restart monasca-log-persister container +- name: Copying over monasca-log-metrics config + vars: + service: "{{ monasca_services['monasca-log-metrics'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-metrics/log-metrics.conf" + mode: "0660" + become: true + register: monasca_log_metrics_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-metrics.conf" + - "{{ node_custom_config }}/monasca/log-metrics.conf" + - "{{ role_path }}/templates/monasca-log-metrics/log-metrics.conf.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-metrics container - name: Check monasca containers become: true diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index 15dbbbdecc..aa233dc2c5 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -7,7 +7,8 @@ when: inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or - inventory_hostname in groups['monasca-log-persister'] + inventory_hostname in groups['monasca-log-persister'] or + inventory_hostname in groups['monasca-log-metrics'] - include_tasks: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -19,4 +20,5 @@ when: inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or - inventory_hostname in groups['monasca-log-persister'] + inventory_hostname in groups['monasca-log-persister'] or + inventory_hostname in groups['monasca-log-metrics'] diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 new file mode 100644 index 0000000000..805164541e --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2 @@ -0,0 +1,76 @@ +# This config file is used to generate Monasca metrics from logs ingested +# by the Monasca Log API. Alarms and notifications can then be set to alert +# users when a particular log message is ingested. This example config file +# generates metrics for all logs which have a log level which isn't +# debug, trace or info level. A user may want to apply additional logic to +# generate special metrics when particular log messages are ingested. For +# example, if HAProxy fails over, file system corruption is detected or +# other scenarios of interest. + +input { + kafka { + zk_connect => "{{ monasca_zookeeper_servers }}" + topic_id => "{{ monasca_transformed_logs_topic }}" + group_id => "log_metrics" + consumer_id => "log_metrics_{{ ansible_hostname }}" + consumer_threads => "{{ monasca_log_pipeline_threads }}" + } +} + +filter { + # Drop everything we don't want to create metrics for. + if ![log][dimensions][log_level] or [log][dimensions][log_level] in [ "debug", "trace", "info" ] { + drop { + } + } + + # Generate a metric name based on the program and log level + mutate { + add_field => { "[metric][name]" => "log.%{[log][dimensions][programname]}.%{[log][dimensions][log_level]}" } + } + + # Form the metric structure. + mutate { + add_field => { "[metric][value]" => 1 } + rename => { "[log][dimensions]" => "[metric][dimensions]" } + rename => { "[metric][dimensions][Hostname]" => "[metric][dimensions][hostname]" } + rename => { "[metric][dimensions][programname]" => "[metric][dimensions][service]" } + } + mutate { + convert => { "[metric][value]" => "float" } + } + + # Convert the timestamp of the event to milliseconds since epoch. + ruby { + code => "event['metric']['timestamp'] = event['@timestamp'].to_i * 1000" + } + + # Clean up any fields which aren't required from the new metric to save space + mutate { + remove_field => ["[metric][dimensions][log_level]", + "[metric][dimensions][domain_id]", + "[metric][dimensions][user_id]", + "[metric][dimensions][tenant_id]", + "[metric][dimensions][project_domain]", + "[metric][dimensions][tag]", + "[metric][dimensions][Logger]", + "[metric][dimensions][Pid]", + "[metric][dimensions][user_domain]", + "[metric][dimensions][request_id]", + "[metric][dimensions][python_module]", + "[metric][meta]", + "creation_time", + "log", + "@version", + "@timestamp"] + } +} + +output { + kafka { + bootstrap_servers => "{{ monasca_kafka_servers }}" + topic_id => "{{ monasca_metrics_topic }}" + client_id => "log_metrics_{{ ansible_hostname }}" + workers => "{{ monasca_log_pipeline_threads }}" + } +} diff --git a/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 new file mode 100644 index 0000000000..287ffcd975 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-metrics/monasca-log-metrics.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-metrics.log -f /etc/logstash/conf.d/log-metrics.conf", + "config_files": [ + { + "source": "{{ container_config_directory }}/log-metrics.conf", + "dest": "/etc/logstash/conf.d/log-metrics.conf", + "owner": "logstash", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/logstash", + "owner": "logstash:kolla", + "recurse": true + } + ] +} diff --git a/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml b/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml new file mode 100644 index 0000000000..8abda7ab34 --- /dev/null +++ b/releasenotes/notes/add-monasca-log-metrics-370846df015ff96a.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add the Monasca Log Metrics service. This service is responsible for + generating metrics from log files.