Merge "Remove Monasca Log Transformer"

ansible/monasca_cleanup.yml

@@ -0,0 +1,14 @@
+---
+- name: Cleanup unused Monasca services
+  hosts:
+    - monasca-api
+    - monasca-grafana
+    - monasca-log-transformer
+    - monasca-log-persister
+    - monasca-log-metrics
+    - monasca-thresh
+    - monasca-notification
+    - monasca-persister
+  roles:
+    - { role: monasca,
+        tags: monasca }

ansible/roles/monasca/defaults/main.yml

@@ -20,10 +20,12 @@ monasca_services:
         mode: "http"
         external: true
         port: "{{ monasca_api_port }}"
+  # NOTE(dszumski): We can remove log_transformer and all other references after the
+  # Wallaby release
   monasca-log-transformer:
     container_name: monasca_log_transformer
     group: monasca-log-transformer
-    enabled: true
+    enabled: false
     image: "{{ monasca_logstash_image_full }}"
     volumes: "{{ monasca_log_transformer_default_volumes + monasca_log_transformer_extra_volumes }}"
     dimensions: "{{ monasca_log_transformer_dimensions }}"
@@ -175,7 +177,7 @@ monasca_all_topics:
   - name: "{{ monasca_transformed_logs_topic }}"
     partitions: "{{ monasca_default_topic_partitions }}"
     replication_factor: "{{ monasca_default_topic_replication_factor }}"
-    enabled: True
+    enabled: False
   - name: "{{ monasca_events_topic }}"
     partitions: "{{ monasca_default_topic_partitions }}"
     replication_factor: "{{ monasca_default_topic_replication_factor }}"

ansible/roles/monasca/handlers/main.yml

@@ -14,21 +14,6 @@
   when:
     - kolla_action != "config"
 
-- name: Restart monasca-log-transformer container
-  vars:
-    service_name: "monasca-log-transformer"
-    service: "{{ monasca_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-
 - name: Restart monasca-log-persister container
   vars:
     service_name: "monasca-log-persister"

ansible/roles/monasca/tasks/cleanup.yml

@@ -0,0 +1,22 @@
+---
+- name: Stop and remove containers for disabled monasca services
+  become: true
+  kolla_docker:
+    action: "stop_and_remove_container"
+    name: "{{ item.value.container_name }}"
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - not item.value.enabled | bool
+  with_dict: "{{ monasca_services }}"
+
+- name: Removing config for any disabled services
+  file:
+    path: "{{ node_config_directory }}/{{ item.key }}"
+    state: "absent"
+  become: true
+  when:
+    - inventory_hostname in groups[item.value.group]
+    - not item.value.enabled | bool
+  with_dict: "{{ monasca_services }}"
+
+# NOTE(dszumski): Docker volume removal is currently a manual procedure

ansible/roles/monasca/tasks/config.yml

@@ -160,29 +160,11 @@
   notify:
     - Restart monasca-api container
 
-- name: Copying over monasca-log-transformer config
-  vars:
-    service: "{{ monasca_services['monasca-log-transformer'] }}"
-  template:
-    src: "{{ item }}"
-    dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf"
-    mode: "0660"
-  become: true
-  with_first_found:
-    - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf"
-    - "{{ node_custom_config }}/monasca/log-transformer.conf"
-    - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2"
-  when:
-    - inventory_hostname in groups[service['group']]
-    - service.enabled | bool
-  notify:
-    - Restart monasca-log-transformer container
-
 - name: Ensuring logstash patterns folder exists
   vars:
-    service: "{{ monasca_services['monasca-log-transformer'] }}"
+    service: "{{ monasca_services['monasca-log-persister'] }}"
   file:
-    path: "{{ node_config_directory }}/monasca-log-transformer/logstash_patterns"
+    path: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns"
     state: "directory"
     mode: "0770"
   become: true
@@ -200,10 +182,10 @@
 
 - name: Copying over custom logstash patterns
   vars:
-    service: "{{ monasca_services['monasca-log-transformer'] }}"
+    service: "{{ monasca_services['monasca-log-persister'] }}"
   template:
     src: "{{ item.path }}"
-    dest: "{{ node_config_directory }}/monasca-log-transformer/logstash_patterns/{{ item.path | basename }}"
+    dest: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns/{{ item.path | basename }}"
     mode: "0660"
   with_items: "{{ monasca_custom_logstash_patterns.files }}"
   become: true
@@ -211,7 +193,7 @@
     - inventory_hostname in groups[service['group']]
     - service.enabled | bool
   notify:
-    - Restart monasca-log-transformer container
+    - Restart monasca-log-persister container
 
 - name: Copying over monasca-log-persister config
   vars:

ansible/roles/monasca/tasks/upgrade.yml

@@ -14,6 +14,8 @@
 
 - import_tasks: config.yml
 
+- import_tasks: cleanup.yml
+
 - import_tasks: check-containers.yml
 
 # NOTE(dszumski): We don't want old Grafana instances running after

ansible/roles/monasca/templates/monasca-log-metrics/log-metrics.conf.j2

@@ -10,7 +10,7 @@
 input {
     kafka {
         bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topics => ["{{ monasca_transformed_logs_topic }}"]
+        topics => ["{{ monasca_raw_logs_topic }}"]
         group_id => "log_metrics"
         consumer_threads => "{{ monasca_log_pipeline_threads }}"
         codec => json

ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2

@@ -1,15 +1,41 @@
-# Persist transformed logs to Elasticsearch
+# Persist logs to Elasticsearch.
 
 input {
     kafka {
         bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topics => ["{{ monasca_transformed_logs_topic }}"]
+        topics => ["{{ monasca_raw_logs_topic }}"]
         group_id => "log_persister"
         consumer_threads => "{{ monasca_log_pipeline_threads }}"
         codec => json
     }
 }
 
+filter {
+    # Update the timestamp of the event based on the time in the message.
+    date {
+        match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss Z", "ISO8601"]
+        remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
+    }
+
+    # Monasca Log API adds a timestamp when it processes a log entry. This
+    # timestamp needs to be converted from seconds since the epoch for
+    # Elasticsearch to parse it correctly. Here we make that conversion.
+    date {
+        match => ["creation_time", "UNIX"]
+        target => "creation_time"
+    }
+
+    # OpenStack log levels are uppercase, and syslog are lowercase.
+    # Furthermore, syslog has more log levels that OpenStack. To avoid
+    # mapping syslog log levels to OpenStack log levels, we standardise
+    # on the syslog style here.
+    if [log][dimensions][log_level] {
+        mutate {
+            lowercase => [ "[log][dimensions][log_level]" ]
+        }
+    }
+}
+
 output {
     elasticsearch {
         index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}"

ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2

@@ -1,46 +0,0 @@
-# Provide input/output streams for transforming Monasca logs.
-# Filters should be provided in other configuration files.
-
-input {
-    kafka {
-        bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topics => ["{{ monasca_raw_logs_topic }}"]
-        group_id => "log_transformer"
-        consumer_threads => "{{ monasca_log_pipeline_threads }}"
-        codec => json
-    }
-}
-
-filter {
-    # Update the timestamp of the event based on the time in the message.
-    date {
-        match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss Z", "ISO8601"]
-        remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
-    }
-
-    # Monasca Log API adds a timestamp when it processes a log entry. This
-    # timestamp needs to be converted from seconds since the epoch for
-    # Elasticsearch to parse it correctly. Here we make that conversion.
-    date {
-        match => ["creation_time", "UNIX"]
-        target => "creation_time"
-    }
-
-    # OpenStack log levels are uppercase, and syslog are lowercase.
-    # Furthermore, syslog has more log levels that OpenStack. To avoid
-    # mapping syslog log levels to OpenStack log levels, we standardise
-    # on the syslog style here.
-    if [log][dimensions][log_level] {
-        mutate {
-            lowercase => [ "[log][dimensions][log_level]" ]
-        }
-    }
-}
-
-output {
-    kafka {
-        codec => json
-        bootstrap_servers => "{{ monasca_kafka_servers }}"
-        topic_id => "{{ monasca_transformed_logs_topic }}"
-    }
-}

ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2

@@ -1,25 +0,0 @@
-{
-    "command": "/usr/share/logstash/bin/logstash --path.settings /etc/logstash/ --log.format json --path.logs /var/log/kolla/logstash/monasca-log-transformer -f /etc/logstash/conf.d/log-transformer.conf",
-    "config_files": [
-        {
-            "source": "{{ container_config_directory }}/log-transformer.conf",
-            "dest": "/etc/logstash/conf.d/log-transformer.conf",
-            "owner": "logstash",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/logstash_patterns/*",
-            "dest": "/etc/logstash/conf.d/patterns/",
-            "owner": "logstash",
-            "perm": "0600",
-            "optional": true
-        }
-    ],
-    "permissions": [
-        {
-            "path": "/var/log/kolla/logstash",
-            "owner": "logstash:kolla",
-            "recurse": true
-        }
-    ]
-}

doc/source/reference/logging-and-monitoring/monasca-guide.rst

@@ -318,6 +318,42 @@ Apply the password changes by running the following command:
 
    kolla-ansible reconfigure -t monasca
 
+Cleanup
+~~~~~~~
+
+From time-to-time it may be necessary to manually invoke the Monasca cleanup
+command. If this is required during an upgrade it will be mentioned in the
+release notes. It may also be necessary to run the cleanup command when
+disabling certain parts of the Monasca pipeline. A full list of scenarios in
+which you must run the cleanup command is given below:
+
+- Upgrading from Victoria to Wallaby to remove the unused Monasca Log
+  Transformer service
+
+The cleanup command can be invoked from the Kolla Ansible CLI, for example:
+
+.. code-block:: console
+
+   kolla-ansible monasca_cleanup
+
+Following cleanup, you may also choose to remove unused container volumes.
+It is recommended to run this manually on each Monasca service host. Note
+that `docker prune` will indiscriminately remove all unused volumes,
+which may not always be what you want. If you wish to keep a subset of
+unused volumes, you can remove them individually.
+
+To remove all unused volumes on a host:
+
+.. code-block:: console
+
+   docker prune
+
+To remove a single unused volume, run for example:
+
+.. code-block:: console
+
+   docker volume rm monasca_log_transformer_data
+
 System requirements and performance impact
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -335,9 +371,8 @@ Monasca will deploy the following Docker containers:
 * Monasca Agent Statsd
 * Monasca API
 * Monasca Log API
-* Monasca Log Transformer (Logstash)
 * Monasca Log Metrics (Logstash)
-* Monasca Log Perister (Logstash)
+* Monasca Log Persister (Logstash)
 * Monasca Notification
 * Monasca Persister
 * Monasca Thresh (Apache Storm topology)

releasenotes/notes/remove-monasca-log-transformer-bdcb3ef1cdbc1611.yaml

@@ -0,0 +1,24 @@
+---
+upgrade:
+  - |
+    Monasca Log Transformer has been merged with Monasca Log Persister to
+    improve performance and reduce resource consumption. Any custom Monasca
+    Log Transformer configuration should be either merged into Monasca Log
+    Persister configuration, or moved outside of the Monasca pipeline, for
+    example, to Fluentd. Any custom Monasca Log Metrics config will also
+    need to be updated to read from the raw logs pipeline, rather than the
+    transformed logs pipeline. The transformed logs pipeline will be
+    removed from Kafka automatically, as will any log transformer containers.
+    There will be a short interruption to logging services whilst the
+    pipeline is updated. During this time it's likely that a small window
+    of logs will be lost from the transformed logs Kafka queue. If this
+    is a problem, the Monasca API should be stopped on all nodes prior to
+    upgrading Monasca. This will allow the transformed logs topic to drain
+    into Elasticsearch before the pipeline is reconfigured. Services such
+    as Fluentd, which post logs to the Monasca API, should buffer logs
+    whilst this happens up to the maximum configured buffer. Note that
+    there may be other services forwarding logs, and these will need to be
+    inspected independently. The Log Transformer volumes will remain on
+    the monitoring nodes and can be manually removed as described in the
+    `documentation
+    <https://docs.openstack.org/kolla-ansible/latest/reference/logging-and-monitoring/monasca-guide.html#cleanup>`__.

tools/kolla-ansible

@@ -133,6 +133,7 @@ Commands:
     mariadb_backup       Take a backup of MariaDB databases
                              --full (default)
                              --incremental
+    monasca_cleanup      Remove unused containers for the Monasca service
     bootstrap-servers    Bootstrap servers with kolla deploy dependencies
     destroy              Destroy Kolla containers, volumes and host configuration
                              --include-images to also destroy Kolla images
@@ -175,6 +176,7 @@ prechecks
 check
 mariadb_recovery
 mariadb_backup
+monasca_cleanup
 bootstrap-servers
 destroy
 deploy
@@ -359,6 +361,11 @@ case "$1" in
         EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE}"
         PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml"
         ;;
+(monasca_cleanup)
+        ACTION="Cleanup unused Monasca services"
+        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=cleanup"
+        PLAYBOOK="${BASEDIR}/ansible/monasca_cleanup.yml"
+        ;;
 (destroy)
         ACTION="Destroy Kolla containers, volumes and host configuration"
         PLAYBOOK="${BASEDIR}/ansible/destroy.yml"