openstack/kolla-ansible
Code Issues Proposed changes
Files
1ddef85973dc510b642a6c633cf99e72320c17b5
kolla-ansible/ansible/roles/monasca/tasks/config.yml
Doug Szumski 0743a9bf4b Remove Monasca Log Transformer 
Historically Monasca Log Transformer has been for log
standardisation and processing. For example, logs from different
sources may use slightly different error levels such as WARN, 5,
or WARNING. Monasca Log Transformer is a place where these could
be 'squashed' into a single error level to simplify log searches
based on labels such as these.

However, in Kolla Ansible, we do this processing in Fluentd so
that the simpler Fluentd -> Elastic -> Kibana pipeline also
benefits. This helps to avoid spreading out log parsing
configuration over many services, with the Fluentd Monasca output
plugin being yet another potential place for processing (which
should be avoided). It therefore makes sense to remove this
service entirely, and squash any existing configuration which
can't be moved to Fluentd into the Log Perister service. I.e.
by removing this pipeline, we don't loose any functionality,
we encourage log processing to take place in Fluentd, or at least
outside of Monasca, and we make significant gains in efficiency
by removing a topic from Kafka which contains a copy of all logs
in transit.

Finally, users forwarding logs from outside the control plane,
eg. from tenant instances, should be encouraged to process the
logs at the point of sending using whichever framework they are
forwarding them with. This makes sense, because all Logstash
configuration in Monasca is only accessible by control plane
admins. A user can't typically do any processing inside Monasca,
with or without this change.

Change-Id: I65c76d0d1cd488725e4233b7e75a11d03866095c
2021-03-03 17:20:18 +00:00

369 lines
12 KiB
YAML
Raw Blame History

---
- name: Ensuring config directories exist
file:
path: "{{ node_config_directory }}/{{ item.key }}"
state: "directory"
owner: "{{ config_owner_user }}"
group: "{{ config_owner_group }}"
mode: "0770"
become: true
when:
- inventory_hostname in groups[item.value.group]
- item.value.enabled | bool
with_dict: "{{ monasca_services }}"
- include_tasks: copy-certs.yml
when:
- kolla_copy_ca_into_containers | bool
- name: Copying over config.json files for services
template:
src: "{{ item.key }}/{{ item.key }}.json.j2"
dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
mode: "0660"
become: true
when:
- inventory_hostname in groups[item.value.group]
- item.value.enabled | bool
with_dict: "{{ monasca_services }}"
notify:
- "Restart {{ item.key }} container"
- name: Copying over monasca-agent-collector config
vars:
service: "{{ monasca_services['monasca-agent-collector'] }}"
merge_yaml:
sources:
- "{{ role_path }}/templates/monasca-agent-collector/{{ item }}.j2"
- "{{ node_custom_config }}/monasca/{{ item }}"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
dest: "{{ node_config_directory }}/monasca-agent-collector/{{ item }}"
mode: "0660"
become: true
with_items:
- agent-collector.yml
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-agent-collector container
- name: Ensuring monasca-agent collector plugin config directory exists
vars:
service: "{{ monasca_services['monasca-agent-collector'] }}"
file:
path: "{{ node_config_directory }}/monasca-agent-collector/plugins"
state: "directory"
owner: "{{ config_owner_user }}"
group: "{{ config_owner_group }}"
mode: "0770"
become: true
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
- name: Find monasca-agent-collector plugin configuration files
find:
paths:
- "{{ role_path }}/templates/monasca-agent-collector/plugins/"
- "{{ node_custom_config }}/monasca/agent_plugins/"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/agent_plugins/"
patterns: '*.yaml'
delegate_to: localhost
register: agent_plugins
- name: Copying over monasca-agent-collector plugins
vars:
service: "{{ monasca_services['monasca-agent-collector'] }}"
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/monasca-agent-collector/plugins/{{ item.path | basename }}"
mode: "0660"
become: true
with_items:
"{{ agent_plugins.files }}"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-agent-collector container
- name: Copying over monasca-agent-forwarder config
vars:
service: "{{ monasca_services['monasca-agent-forwarder'] }}"
merge_yaml:
sources:
- "{{ role_path }}/templates/monasca-agent-forwarder/{{ item }}.j2"
- "{{ node_custom_config }}/monasca/{{ item }}"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
dest: "{{ node_config_directory }}/monasca-agent-forwarder/{{ item }}"
mode: "0660"
become: true
with_items:
- agent-forwarder.yml
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-agent-forwarder container
- name: Copying over monasca-agent-statsd config
vars:
service: "{{ monasca_services['monasca-agent-statsd'] }}"
merge_yaml:
sources:
- "{{ role_path }}/templates/monasca-agent-statsd/{{ item }}.j2"
- "{{ node_custom_config }}/monasca/{{ item }}"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
dest: "{{ node_config_directory }}/monasca-agent-statsd/{{ item }}"
mode: "0660"
become: true
with_items:
- agent-statsd.yml
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-agent-statsd container
- name: Copying over monasca-api config
vars:
service: "{{ monasca_services['monasca-api'] }}"
merge_configs:
sources:
- "{{ role_path }}/templates/monasca-api/{{ item }}.j2"
- "{{ node_custom_config }}/monasca/{{ item }}"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
dest: "{{ node_config_directory }}/monasca-api/{{ item }}"
mode: "0660"
become: true
with_items:
- api.conf
- api-config.ini
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-api container
- name: Copying over monasca-api wsgi config
vars:
service: "{{ monasca_services['monasca-api'] }}"
template:
src: "{{ role_path }}/templates/monasca-api/wsgi-api.conf.j2"
dest: "{{ node_config_directory }}/monasca-api/wsgi-api.conf"
mode: "0660"
become: true
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-api container
- name: Ensuring logstash patterns folder exists
vars:
service: "{{ monasca_services['monasca-log-persister'] }}"
file:
path: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns"
state: "directory"
mode: "0770"
become: true
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
- name: Find custom logstash patterns
find:
path: "{{ node_custom_config }}/monasca/logstash_patterns"
pattern: "*"
delegate_to: localhost
run_once: True
register: monasca_custom_logstash_patterns
- name: Copying over custom logstash patterns
vars:
service: "{{ monasca_services['monasca-log-persister'] }}"
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/monasca-log-persister/logstash_patterns/{{ item.path | basename }}"
mode: "0660"
with_items: "{{ monasca_custom_logstash_patterns.files }}"
become: true
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-log-persister container
- name: Copying over monasca-log-persister config
vars:
service: "{{ monasca_services['monasca-log-persister'] }}"
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/monasca-log-persister/log-persister.conf"
mode: "0660"
become: true
with_first_found:
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-persister.conf"
- "{{ node_custom_config }}/monasca/log-persister.conf"
- "{{ role_path }}/templates/monasca-log-persister/log-persister.conf.j2"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-log-persister container
- name: Copying over monasca-log-persister elasticsearch template
vars:
service: "{{ monasca_services['monasca-log-persister'] }}"
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/monasca-log-persister/elasticsearch-template.json"
mode: "0660"
become: true
with_first_found:
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/elasticsearch-template.json"
- "{{ node_custom_config }}/monasca/elasticsearch-template.json"
- "{{ role_path }}/templates/monasca-log-persister/elasticsearch-template.json"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-log-persister container
- name: Copying over monasca-log-metrics config
vars:
service: "{{ monasca_services['monasca-log-metrics'] }}"
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/monasca-log-metrics/log-metrics.conf"
mode: "0660"
become: true
with_first_found:
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-metrics.conf"
- "{{ node_custom_config }}/monasca/log-metrics.conf"
- "{{ role_path }}/templates/monasca-log-metrics/log-metrics.conf.j2"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-log-metrics container
- name: Copying over monasca-thresh config
vars:
service: "{{ monasca_services['monasca-thresh'] }}"
# NOTE(dszumski): We can't use merge_yaml since it replaces empty values
# with `null`. This breaks the thresholder config file parsing (which should
# probably be more robust).
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/monasca-thresh/thresh-config.yml"
mode: "0660"
become: true
with_first_found:
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/thresh-config.yml"
- "{{ node_custom_config }}/monasca/thresh-config.yml"
- "{{ role_path }}/templates/monasca-thresh/thresh-config.yml.j2"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-thresh container
- name: Copying over monasca-thresh storm config
vars:
service: "{{ monasca_services['monasca-thresh'] }}"
template:
src: "{{ item }}"
dest: "{{ node_config_directory }}/monasca-thresh/storm.yml"
mode: "0660"
become: true
with_first_found:
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/storm.yml"
- "{{ node_custom_config }}/monasca/storm.yml"
- "{{ role_path }}/templates/monasca-thresh/storm.yml.j2"
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-thresh container
- name: Copying over monasca-notification config
vars:
service: "{{ monasca_services['monasca-notification'] }}"
merge_configs:
sources:
- "{{ role_path }}/templates/monasca-notification/{{ item }}.j2"
- "{{ node_custom_config }}/monasca/{{ item }}"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
dest: "{{ node_config_directory }}/monasca-notification/{{ item }}"
mode: "0660"
become: true
with_items:
- notification.conf
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-notification container
- name: Check for monasca-notification templates
stat:
path: "{{ node_custom_config }}/monasca/notification_templates"
delegate_to: localhost
run_once: True
register: notification_templates
- name: Copying over monasca-notification templates
vars:
service: "{{ monasca_services['monasca-notification'] }}"
copy:
src: "{{ node_custom_config }}/monasca/notification_templates"
dest: "{{ node_config_directory }}/monasca-notification/"
mode: "0660"
become: true
when:
- notification_templates.stat.exists and notification_templates.stat.isdir
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-notification container
- name: Copying over monasca-persister config
vars:
service: "{{ monasca_services['monasca-persister'] }}"
merge_configs:
sources:
- "{{ role_path }}/templates/monasca-persister/{{ item }}.j2"
- "{{ node_custom_config }}/monasca/{{ item }}"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
dest: "{{ node_config_directory }}/monasca-persister/{{ item }}"
mode: "0660"
become: true
with_items:
- persister.conf
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-persister container
- name: Copying over monasca-grafana config file
vars:
service: "{{ monasca_services['monasca-grafana'] }}"
merge_configs:
sources:
- "{{ role_path }}/templates/monasca-grafana/{{ item }}.j2"
- "{{ node_custom_config }}/monasca/{{ item }}"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
dest: "{{ node_config_directory }}/monasca-grafana/{{ item }}"
mode: "0660"
become: true
with_items:
- grafana.ini
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-grafana container
View Git Blame Copy Permalink