diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 0eb6b6641d..845b5c03ad 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -446,6 +446,9 @@ monasca [monasca-log-api:children] monasca +[monasca-log-transformer:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 1a223722c5..1466025b75 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -455,6 +455,9 @@ monasca [monasca-log-api:children] monasca +[monasca-log-transformer:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index b5fc75a695..4db2feb1c6 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -18,6 +18,15 @@ monasca_services: - "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" + monasca-log-transformer: + container_name: monasca_log_transformer + group: monasca-log-transformer + enabled: true + image: "{{ monasca_logstash_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" #################### # Databases @@ -34,10 +43,12 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}" # Monasca #################### monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}" +monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_metrics_topic: "metrics" monasca_raw_logs_topic: "logs" +monasca_transformed_logs_topic: "transformed-logs" #################### # Docker @@ -53,6 +64,10 @@ monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ monasca_log_api_tag: "{{ monasca_tag }}" monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}" +monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-logstash" +monasca_logstash_tag: "{{ monasca_tag }}" +monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}" + #################### # OpenStack #################### diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index dffdbc7d59..6caadb6df0 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -42,3 +42,23 @@ or monasca_log_api_confs.changed | bool or monasca_log_api_wsgi_confs.changed | bool or monasca_log_api_container.changed | bool + +- name: Restart monasca-log-transformer container + vars: + service_name: "monasca-log-transformer" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_log_transformer_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_log_transformer_confs.changed | bool + or monasca_log_transformer_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 164b970868..42c40f38dc 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -25,8 +25,7 @@ - item.value.enabled | bool with_dict: "{{ monasca_services }}" notify: - - Restart monasca-api container - - Restart monasca-log-api container + - "Restart {{ item.key }} container" - name: Copying over monasca-api config vars: @@ -100,6 +99,25 @@ notify: - Restart monasca-log-api container +- name: Copying over monasca-log-transformer config + vars: + service: "{{ monasca_services['monasca-log-transformer'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-transformer/log-transformer.conf" + mode: "0660" + become: true + register: monasca_log_transformer_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-transformer.conf" + - "{{ node_custom_config }}/monasca/log-transformer.conf" + - "{{ role_path }}/templates/monasca-log-transformer/log-transformer.conf.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-transformer container + - name: Check monasca containers become: true kolla_docker: @@ -115,5 +133,4 @@ - item.value.enabled | bool with_dict: "{{ monasca_services }}" notify: - - Restart monasca-api container - - Restart monasca-log-api container + - "Restart {{ item.key }} container" diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index decfa5ff9f..b386d15e3f 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -5,7 +5,8 @@ - include: config.yml when: inventory_hostname in groups['monasca-api'] or - inventory_hostname in groups['monasca-log-api'] + inventory_hostname in groups['monasca-log-api'] or + inventory_hostname in groups['monasca-log-transformer'] - include: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -15,4 +16,5 @@ - include: check.yml when: inventory_hostname in groups['monasca-api'] or - inventory_hostname in groups['monasca-log-api'] + inventory_hostname in groups['monasca-log-api'] or + inventory_hostname in groups['monasca-log-transformer'] diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 new file mode 100644 index 0000000000..c571623317 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2 @@ -0,0 +1,25 @@ +# Provide input/output streams for transforming Monasca logs. +# Filters should be provided in other configuration files. + +input { + kafka { + zk_connect => "{{ monasca_zookeeper_servers }}" + topic_id => "{{ monasca_raw_logs_topic }}" + group_id => "transformer-logstash-consumer" + } +} + +filter { + # Update the timestamp of the event based on the time in the message. + date { + match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"] + remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ] + } +} + +output { + kafka { + bootstrap_servers => "{{ monasca_kafka_servers }}" + topic_id => "{{ monasca_transformed_logs_topic }}" + } +} diff --git a/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 new file mode 100644 index 0000000000..94cb1cf375 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2 @@ -0,0 +1,18 @@ +{ + "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf", + "config_files": [ + { + "source": "{{ container_config_directory }}/log-transformer.conf", + "dest": "/etc/logstash/conf.d/log-transformer.conf", + "owner": "logstash", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/logstash", + "owner": "logstash:kolla", + "recurse": true + } + ] +} diff --git a/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml b/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml new file mode 100644 index 0000000000..3e1eb09639 --- /dev/null +++ b/releasenotes/notes/add-monasca-log-transformer-80d985fc77603478.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for deploying the Monasca Log Transformer for + providing log standardisation in Monasca.