Files
kolla-ansible/ansible/roles/common/tasks/config.yml
Mark Goddard f329af7dfa Performance: use a single config file for logrotate
Currently we generate a logrotate configuration file for each enabled
service. These are then included from a logrotate.d directory. With a
large number of hosts, this can take a long time to template.

Benchmarking of templating is available at [1].

This change switches to a single logrotate configuration file for all
services, with the include done locally using jinja. This should
drastically improve the performance of this task.

[1] https://github.com/stackhpc/ansible-scaling/blob/master/doc/template.md

Partially-Implements: blueprint performance-improvements

Change-Id: I39cfa70bef6560f615cad516c43aaef6a523b964
2020-07-28 12:14:47 +01:00

406 lines
14 KiB
YAML

---
- name: Ensuring config directories exist
vars:
service_name: "{{ item.0.service_name }}"
service: "{{ common_services[service_name] }}"
file:
path: "{{ node_config_directory }}/{{ item.1 }}"
state: "directory"
owner: "{{ config_owner_user }}"
group: "{{ config_owner_group }}"
mode: "0770"
become: true
with_subelements:
- - service_name: "cron"
paths:
- "cron"
- "cron/logrotate"
- service_name: "fluentd"
paths:
- "fluentd"
- "fluentd/input"
- "fluentd/output"
- "fluentd/format"
- "fluentd/filter"
- service_name: "kolla-toolbox"
paths:
- "kolla-toolbox"
- paths
when: service | service_enabled_and_mapped_to_host
- name: Ensure fluentd image is present for label check
vars:
service_name: "fluentd"
service: "{{ common_services[service_name] }}"
become: true
kolla_docker:
action: "ensure_image"
common_options: "{{ docker_common_options }}"
image: "{{ service.image }}"
when: service | service_enabled_and_mapped_to_host
- name: Fetch fluentd image labels
vars:
service_name: "fluentd"
service: "{{ common_services[service_name] }}"
become: true
docker_image_info:
name: "{{ service.image }}"
register: fluentd_labels
when: service | service_enabled_and_mapped_to_host
- name: Set fluentd facts
set_fact:
fluentd_binary: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}"
when: common_services.fluentd | service_enabled_and_mapped_to_host
- include_tasks: copy-certs.yml
when:
- kolla_copy_ca_into_containers | bool
- name: Copying over config.json files for services
template:
src: "{{ item.key }}.json.j2"
dest: "{{ node_config_directory }}/{{ item.key }}/config.json"
mode: "0660"
become: true
when: item.value | service_enabled_and_mapped_to_host
with_dict: "{{ common_services }}"
notify:
- "Restart {{ item.key }} container"
- name: Find custom fluentd input config files
find:
path: "{{ node_custom_config }}/fluentd/input"
pattern: "*.conf"
run_once: True
register: find_custom_fluentd_inputs
delegate_to: localhost
when: common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over fluentd input config files
vars:
customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | map('basename') | list }}"
template:
src: "conf/input/{{ item }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/input/{{ item }}.conf"
mode: "0660"
become: true
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item ~ '.conf' not in customised_input_files
with_items:
- "00-global"
- "01-syslog"
- "02-mariadb"
- "03-rabbitmq"
- "04-openstack-wsgi"
- "05-libvirt"
- "06-zookeeper"
- "07-kafka"
- "09-monasca"
notify:
- Restart fluentd container
- name: Copying over custom fluentd input config files
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/fluentd/input/{{ item.path | basename }}"
mode: "0660"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
with_items: "{{ find_custom_fluentd_inputs.files }}"
notify:
- Restart fluentd container
- name: Determine whether logs should be forwarded directly to Elasticsearch
set_fact:
log_direct_to_elasticsearch: "{{ ( enable_elasticsearch | bool or
( elasticsearch_address != kolla_internal_vip_address )) and
not enable_monasca | bool }}"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- name: Find custom fluentd output config files
find:
path: "{{ node_custom_config }}/fluentd/output"
pattern: "*.conf"
run_once: True
register: find_custom_fluentd_outputs
delegate_to: localhost
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over fluentd output config files
vars:
customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | map('basename') | list }}"
template:
src: "conf/output/{{ item.name }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf"
mode: "0660"
become: true
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item.enabled | bool
- item.name ~ '.conf' not in customised_output_files
with_items:
- name: "00-local"
enabled: true
- name: "01-es"
enabled: "{{ log_direct_to_elasticsearch }}"
- name: "02-monasca"
enabled: "{{ enable_monasca | bool }}"
notify:
- Restart fluentd container
- name: Removing stale output config files
file:
path: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf"
state: "absent"
become: true
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item.disable | bool
with_items:
- name: "02-monasca"
disable: "{{ not enable_monasca | bool }}"
- name: "01-es"
disable: "{{ not log_direct_to_elasticsearch }}"
notify:
- Restart fluentd container
- name: Copying over custom fluentd output config files
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/fluentd/output/{{ item.path | basename }}"
mode: "0660"
become: true
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
with_items: "{{ find_custom_fluentd_outputs.files }}"
notify:
- Restart fluentd container
- name: Find custom fluentd format config files
find:
path: "{{ node_custom_config }}/fluentd/format"
pattern: "*.conf"
run_once: True
register: find_custom_fluentd_format
delegate_to: localhost
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over fluentd format config files
vars:
customised_format_files: "{{ find_custom_fluentd_format.files | map(attribute='path') | map('basename') | list }}"
template:
src: "conf/format/{{ item }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/format/{{ item }}.conf"
mode: "0660"
become: true
with_items:
- "apache_access"
- "wsgi_access"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item ~ '.conf' not in customised_format_files
notify:
- Restart fluentd container
- name: Copying over custom fluentd format config files
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/fluentd/format/{{ item.path | basename }}"
mode: "0660"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
with_items: "{{ find_custom_fluentd_format.files }}"
notify:
- Restart fluentd container
- name: Find custom fluentd filter config files
find:
path: "{{ node_custom_config }}/fluentd/filter"
pattern: "*.conf"
run_once: True
register: find_custom_fluentd_filters
delegate_to: localhost
when: common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over fluentd filter config files
vars:
customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}"
fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
template:
src: "conf/filter/{{ item.src }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf"
mode: "0660"
become: true
with_items:
- src: 00-record_transformer
dest: 00-record_transformer
- src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}"
dest: 01-rewrite
- src: 02-parser
dest: 02-parser
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item.src ~ '.conf' not in customised_filter_files
notify:
- Restart fluentd container
- name: Copying over custom fluentd filter config files
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/fluentd/filter/{{ item.path | basename }}"
mode: "0660"
become: true
with_items: "{{ find_custom_fluentd_filters.files }}"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
notify:
- Restart fluentd container
- name: Copying over td-agent.conf
template:
src: "td-agent.conf.j2"
dest: "{{ node_config_directory }}/{{ item }}/td-agent.conf"
mode: "0660"
become: true
with_items:
- "fluentd"
when: common_services.fluentd | service_enabled_and_mapped_to_host
notify:
- Restart fluentd container
- name: Copying over cron logrotate config file
vars:
cron_logrotate_enabled_services: >-
{{ cron_logrotate_services |
selectattr('enabled') |
map(attribute='name') |
list }}
cron_logrotate_services:
- { name: "ansible", enabled: "yes" }
- { name: "aodh", enabled: "{{ enable_aodh | bool }}" }
- { name: "barbican", enabled: "{{ enable_barbican | bool }}" }
- { name: "blazar", enabled: "{{ enable_blazar | bool }}" }
- { name: "ceilometer", enabled: "{{ enable_ceilometer | bool }}" }
- { name: "chrony", enabled: "{{ enable_chrony | bool }}" }
- { name: "cinder", enabled: "{{ enable_cinder | bool }}" }
- { name: "cloudkitty", enabled: "{{ enable_cloudkitty | bool }}" }
- { name: "collectd", enabled: "{{ enable_collectd | bool }}" }
- { name: "cyborg", enabled: "{{ enable_cyborg | bool }}" }
- { name: "designate", enabled: "{{ enable_designate | bool }}" }
- { name: "elasticsearch", enabled: "{{ enable_elasticsearch | bool }}" }
- { name: "etcd", enabled: "{{ enable_etcd | bool }}" }
- { name: "fluentd", enabled: "{{ enable_fluentd | bool }}" }
- { name: "freezer", enabled: "{{ enable_freezer | bool }}" }
- { name: "glance", enabled: "{{ enable_glance | bool }}" }
- { name: "glance-tls-proxy", enabled: "{{ enable_glance | bool }}" }
- { name: "gnocchi", enabled: "{{ enable_gnocchi | bool }}" }
- { name: "grafana", enabled: "{{ enable_grafana | bool }}" }
- { name: "haproxy", enabled: "{{ enable_haproxy | bool }}" }
- { name: "heat", enabled: "{{ enable_heat | bool }}" }
- { name: "horizon", enabled: "{{ enable_horizon | bool }}" }
- { name: "influxdb", enabled: "{{ enable_influxdb | bool }}" }
- { name: "ironic", enabled: "{{ enable_ironic | bool }}" }
- { name: "ironic-inspector", enabled: "{{ enable_ironic | bool }}" }
- { name: "iscsid", enabled: "{{ enable_iscsid | bool }}" }
- { name: "kafka", enabled: "{{ enable_kafka | bool }}" }
- { name: "karbor", enabled: "{{ enable_karbor | bool }}" }
- { name: "keepalived", enabled: "{{ enable_haproxy | bool }}" }
- { name: "keystone", enabled: "{{ enable_keystone | bool }}" }
- { name: "kibana", enabled: "{{ enable_kibana | bool }}" }
- { name: "kuryr", enabled: "{{ enable_kuryr | bool }}" }
- { name: "magnum", enabled: "{{ enable_magnum | bool }}" }
- { name: "manila", enabled: "{{ enable_manila | bool }}" }
- { name: "mariadb", enabled: "{{ enable_mariadb | bool }}" }
- { name: "masakari", enabled: "{{ enable_masakari | bool }}" }
- { name: "mistral", enabled: "{{ enable_mistral | bool }}" }
- { name: "monasca", enabled: "{{ enable_monasca | bool }}" }
- { name: "murano", enabled: "{{ enable_murano | bool }}" }
- { name: "neutron", enabled: "{{ enable_neutron | bool }}" }
- { name: "nova", enabled: "{{ enable_nova | bool }}" }
- { name: "octavia", enabled: "{{ enable_octavia | bool }}" }
- { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq | bool }}" }
- { name: "panko", enabled: "{{ enable_panko | bool }}" }
- { name: "qinling", enabled: "{{ enable_qinling | bool }}" }
- { name: "rabbitmq", enabled: "{{ enable_rabbitmq | bool }}" }
- { name: "rally", enabled: "{{ enable_rally | bool }}" }
- { name: "sahara", enabled: "{{ enable_sahara | bool }}" }
- { name: "searchlight", enabled: "{{ enable_searchlight | bool }}" }
- { name: "senlin", enabled: "{{ enable_senlin | bool }}" }
- { name: "skydive", enabled: "{{ enable_skydive | bool }}" }
- { name: "solum", enabled: "{{ enable_solum | bool }}" }
- { name: "storm", enabled: "{{ enable_storm | bool }}" }
- { name: "swift", enabled: "{{ enable_swift | bool }}" }
- { name: "tacker", enabled: "{{ enable_tacker | bool }}" }
- { name: "tempest", enabled: "{{ enable_tempest | bool }}" }
- { name: "trove", enabled: "{{ enable_trove | bool }}" }
- { name: "vitrage", enabled: "{{ enable_vitrage | bool }}" }
- { name: "watcher", enabled: "{{ enable_watcher | bool }}" }
- { name: "zookeeper", enabled: "{{ enable_zookeeper | bool }}" }
- { name: "zun", enabled: "{{ enable_zun | bool }}" }
template:
src: "cron-logrotate-global.conf.j2"
dest: "{{ node_config_directory }}/cron/logrotate.conf"
mode: "0660"
become: true
when:
- common_services.cron | service_enabled_and_mapped_to_host
notify:
- Restart cron container
- name: Ensure RabbitMQ Erlang cookie exists
become: true
copy:
content: "{{ rabbitmq_cluster_cookie }}"
dest: "{{ node_config_directory }}/kolla-toolbox/rabbitmq-erlang.cookie"
mode: "0660"
when:
- common_services['kolla-toolbox'] | service_enabled_and_mapped_to_host
- enable_rabbitmq | bool
notify:
- Restart kolla-toolbox container
- name: Ensuring config directories have correct owner and permission
become: true
file:
path: "{{ node_config_directory }}/{{ item.key }}"
owner: "{{ config_owner_user }}"
group: "{{ config_owner_group }}"
mode: "0770"
ignore_errors: "{{ ansible_check_mode }}"
when:
- item.value | service_enabled_and_mapped_to_host
- item.key != "kolla-toolbox"
with_dict: "{{ common_services }}"
- name: Copy rabbitmq-env.conf to kolla toolbox
copy:
content: |
RABBITMQ_CTL_ERL_ARGS="-proto_dist inet6_tcp"
export ERL_INETRC=/etc/rabbitmq/erl_inetrc
dest: "{{ node_config_directory }}/kolla-toolbox/rabbitmq-env.conf"
mode: "0600"
become: true
when:
- common_services['kolla-toolbox'] | service_enabled_and_mapped_to_host
- api_address_family == "ipv6"
- name: Copy rabbitmq erl_intr to kolla toolbox
copy:
content: |
{inet6,true}.
dest: "{{ node_config_directory }}/kolla-toolbox/erl_inetrc"
mode: "0600"
become: true
when:
- common_services['kolla-toolbox'] | service_enabled_and_mapped_to_host
- api_address_family == "ipv6"
- include_tasks: check-containers.yml
when: kolla_action != "config"