Support deploying Monasca Log Transformer

The Monasca Log Transformer takes raw, unstandardised logs from one
Kafka topic, standardises them with whatever rules the operator wants
to use, and then writes them to a standardised logs topic in Kafka. It
is currently implemented as a Logstash config file.

Since Kolla does a fairly good job of standardising logs, this service
does very little processing. However, when other sources of logs
are used, it may be useful to add rules to the Transformer, particularly
if it's not possible to standardise the logs at source.

Ingesting the logs from this service with Fluentd will be covered under
a different commit.

Change-Id: I31cbb7e9a40a848391f517a56a67e3fd5bc12529
Partially-Implements: blueprint monasca-roles

ansible/inventory/all-in-one

@@ -446,6 +446,9 @@ monasca
 [monasca-log-api:children]
 monasca
 
+[monasca-log-transformer:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/inventory/multinode

@@ -455,6 +455,9 @@ monasca
 [monasca-log-api:children]
 monasca
 
+[monasca-log-transformer:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/roles/monasca/defaults/main.yml

@@ -18,6 +18,15 @@ monasca_services:
       - "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
+  monasca-log-transformer:
+    container_name: monasca_log_transformer
+    group: monasca-log-transformer
+    enabled: true
+    image: "{{ monasca_logstash_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
 
 ####################
 # Databases
@@ -34,10 +43,12 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}"
 # Monasca
 ####################
 monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 
 monasca_metrics_topic: "metrics"
 monasca_raw_logs_topic: "logs"
+monasca_transformed_logs_topic: "transformed-logs"
 
 ####################
 # Docker
@@ -53,6 +64,10 @@ monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{
 monasca_log_api_tag: "{{ monasca_tag }}"
 monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}"
 
+monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-logstash"
+monasca_logstash_tag: "{{ monasca_tag }}"
+monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}"
+
 ####################
 # OpenStack
 ####################

ansible/roles/monasca/handlers/main.yml

@@ -42,3 +42,23 @@
       or monasca_log_api_confs.changed | bool
       or monasca_log_api_wsgi_confs.changed | bool
       or monasca_log_api_container.changed | bool
+
+- name: Restart monasca-log-transformer container
+  vars:
+    service_name: "monasca-log-transformer"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_log_transformer_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_log_transformer_confs.changed | bool
+      or monasca_log_transformer_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -25,8 +25,7 @@
     - item.value.enabled | bool
   with_dict: "{{ monasca_services }}"
   notify:
-    - Restart monasca-api container
+    - "Restart {{ item.key }} container"
-    - Restart monasca-log-api container
 
 - name: Copying over monasca-api config
   vars:
@@ -100,6 +99,25 @@
   notify:
     - Restart monasca-log-api container
 
+- name: Copying over monasca-log-transformer config
+  vars:
+    service: "{{ monasca_services['monasca-log-transformer'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf"
+    mode: "0660"
+  become: true
+  register: monasca_log_transformer_confs
+  with_first_found:
+    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf"
+    - "{{ node_custom_config }}/monasca/log-transformer.conf"
+    - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2"
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-transformer container
+
 - name: Check monasca containers
   become: true
   kolla_docker:
@@ -115,5 +133,4 @@
     - item.value.enabled | bool
   with_dict: "{{ monasca_services }}"
   notify:
-    - Restart monasca-api container
+    - "Restart {{ item.key }} container"
-    - Restart monasca-log-api container

ansible/roles/monasca/tasks/deploy.yml

@@ -5,7 +5,8 @@
 
 - include: config.yml
   when: inventory_hostname in groups['monasca-api'] or
-        inventory_hostname in groups['monasca-log-api']
+        inventory_hostname in groups['monasca-log-api'] or
+        inventory_hostname in groups['monasca-log-transformer']
 
 - include: bootstrap.yml
   when: inventory_hostname in groups['monasca-api']
@@ -15,4 +16,5 @@
 
 - include: check.yml
   when: inventory_hostname in groups['monasca-api'] or
-        inventory_hostname in groups['monasca-log-api']
+        inventory_hostname in groups['monasca-log-api'] or
+        inventory_hostname in groups['monasca-log-transformer']

ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2

@@ -0,0 +1,25 @@
+# Provide input/output streams for transforming Monasca logs.
+# Filters should be provided in other configuration files.
+
+input {
+    kafka {
+        zk_connect => "{{ monasca_zookeeper_servers }}"
+        topic_id => "{{ monasca_raw_logs_topic }}"
+        group_id => "transformer-logstash-consumer"
+    }
+}
+
+filter {
+    # Update the timestamp of the event based on the time in the message.
+    date {
+        match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"]
+        remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
+    }
+}
+
+output {
+    kafka {
+        bootstrap_servers => "{{ monasca_kafka_servers }}"
+        topic_id => "{{ monasca_transformed_logs_topic }}"
+    }
+}

ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/log-transformer.conf",
+            "dest": "/etc/logstash/conf.d/log-transformer.conf",
+            "owner": "logstash",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/logstash",
+            "owner": "logstash:kolla",
+            "recurse": true
+        }
+    ]
+}

releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add support for deploying the Monasca Log Transformer for
+    providing log standardisation in Monasca.