openstack/kolla-ansible
Code Issues Proposed changes

Performance: use a single config file for logrotate

Browse Source 
Currently we generate a logrotate configuration file for each enabled
service. These are then included from a logrotate.d directory. With a
large number of hosts, this can take a long time to template.

Benchmarking of templating is available at [1].

This change switches to a single logrotate configuration file for all
services, with the include done locally using jinja. This should
drastically improve the performance of this task.

[1] https://github.com/stackhpc/ansible-scaling/blob/master/doc/template.md

Partially-Implements: blueprint performance-improvements

Change-Id: I39cfa70bef6560f615cad516c43aaef6a523b964
This commit is contained in:
Mark Goddard 2020-06-30 12:27:13 +01:00
parent da1d14b186
commit f329af7dfa
4 changed files with 82 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
ansible/roles/common/tasks/config.yml
View File

@ -274,78 +274,82 @@
notify:
- Restart fluentd container
- name: Copying over cron logrotate config files
- name: Copying over cron logrotate config file
vars:
cron_logrotate_enabled_services: >-
{{ cron_logrotate_services |
selectattr('enabled') |
map(attribute='name') |
list }}
cron_logrotate_services:
- { name: "ansible", enabled: "yes" }
- { name: "aodh", enabled: "{{ enable_aodh | bool }}" }
- { name: "barbican", enabled: "{{ enable_barbican | bool }}" }
- { name: "blazar", enabled: "{{ enable_blazar | bool }}" }
- { name: "ceilometer", enabled: "{{ enable_ceilometer | bool }}" }
- { name: "chrony", enabled: "{{ enable_chrony | bool }}" }
- { name: "cinder", enabled: "{{ enable_cinder | bool }}" }
- { name: "cloudkitty", enabled: "{{ enable_cloudkitty | bool }}" }
- { name: "collectd", enabled: "{{ enable_collectd | bool }}" }
- { name: "cyborg", enabled: "{{ enable_cyborg | bool }}" }
- { name: "designate", enabled: "{{ enable_designate | bool }}" }
- { name: "elasticsearch", enabled: "{{ enable_elasticsearch | bool }}" }
- { name: "etcd", enabled: "{{ enable_etcd | bool }}" }
- { name: "fluentd", enabled: "{{ enable_fluentd | bool }}" }
- { name: "freezer", enabled: "{{ enable_freezer | bool }}" }
- { name: "glance", enabled: "{{ enable_glance | bool }}" }
- { name: "glance-tls-proxy", enabled: "{{ enable_glance | bool }}" }
- { name: "gnocchi", enabled: "{{ enable_gnocchi | bool }}" }
- { name: "grafana", enabled: "{{ enable_grafana | bool }}" }
- { name: "haproxy", enabled: "{{ enable_haproxy | bool }}" }
- { name: "heat", enabled: "{{ enable_heat | bool }}" }
- { name: "horizon", enabled: "{{ enable_horizon | bool }}" }
- { name: "influxdb", enabled: "{{ enable_influxdb | bool }}" }
- { name: "ironic", enabled: "{{ enable_ironic | bool }}" }
- { name: "ironic-inspector", enabled: "{{ enable_ironic | bool }}" }
- { name: "iscsid", enabled: "{{ enable_iscsid | bool }}" }
- { name: "kafka", enabled: "{{ enable_kafka | bool }}" }
- { name: "karbor", enabled: "{{ enable_karbor | bool }}" }
- { name: "keepalived", enabled: "{{ enable_haproxy | bool }}" }
- { name: "keystone", enabled: "{{ enable_keystone | bool }}" }
- { name: "kibana", enabled: "{{ enable_kibana | bool }}" }
- { name: "kuryr", enabled: "{{ enable_kuryr | bool }}" }
- { name: "magnum", enabled: "{{ enable_magnum | bool }}" }
- { name: "manila", enabled: "{{ enable_manila | bool }}" }
- { name: "mariadb", enabled: "{{ enable_mariadb | bool }}" }
- { name: "masakari", enabled: "{{ enable_masakari | bool }}" }
- { name: "mistral", enabled: "{{ enable_mistral | bool }}" }
- { name: "monasca", enabled: "{{ enable_monasca | bool }}" }
- { name: "murano", enabled: "{{ enable_murano | bool }}" }
- { name: "neutron", enabled: "{{ enable_neutron | bool }}" }
- { name: "nova", enabled: "{{ enable_nova | bool }}" }
- { name: "octavia", enabled: "{{ enable_octavia | bool }}" }
- { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq | bool }}" }
- { name: "panko", enabled: "{{ enable_panko | bool }}" }
- { name: "qinling", enabled: "{{ enable_qinling | bool }}" }
- { name: "rabbitmq", enabled: "{{ enable_rabbitmq | bool }}" }
- { name: "rally", enabled: "{{ enable_rally | bool }}" }
- { name: "sahara", enabled: "{{ enable_sahara | bool }}" }
- { name: "searchlight", enabled: "{{ enable_searchlight | bool }}" }
- { name: "senlin", enabled: "{{ enable_senlin | bool }}" }
- { name: "skydive", enabled: "{{ enable_skydive | bool }}" }
- { name: "solum", enabled: "{{ enable_solum | bool }}" }
- { name: "storm", enabled: "{{ enable_storm | bool }}" }
- { name: "swift", enabled: "{{ enable_swift | bool }}" }
- { name: "tacker", enabled: "{{ enable_tacker | bool }}" }
- { name: "tempest", enabled: "{{ enable_tempest | bool }}" }
- { name: "trove", enabled: "{{ enable_trove | bool }}" }
- { name: "vitrage", enabled: "{{ enable_vitrage | bool }}" }
- { name: "watcher", enabled: "{{ enable_watcher | bool }}" }
- { name: "zookeeper", enabled: "{{ enable_zookeeper | bool }}" }
- { name: "zun", enabled: "{{ enable_zun | bool }}" }
template:
src: "cron-logrotate-{{ item.name }}.conf.j2"
dest: "{{ node_config_directory }}/cron/logrotate/{{ item.name }}.conf"
src: "cron-logrotate-global.conf.j2"
dest: "{{ node_config_directory }}/cron/logrotate.conf"
mode: "0660"
become: true
when:
- common_services.cron | service_enabled_and_mapped_to_host
- item.enabled | bool
with_items:
- { name: "ansible", enabled: "yes" }
- { name: "aodh", enabled: "{{ enable_aodh }}" }
- { name: "barbican", enabled: "{{ enable_barbican }}" }
- { name: "blazar", enabled: "{{ enable_blazar }}" }
- { name: "ceilometer", enabled: "{{ enable_ceilometer }}" }
- { name: "chrony", enabled: "{{ enable_chrony }}" }
- { name: "cinder", enabled: "{{ enable_cinder }}" }
- { name: "cloudkitty", enabled: "{{ enable_cloudkitty }}" }
- { name: "collectd", enabled: "{{ enable_collectd }}" }
- { name: "cyborg", enabled: "{{ enable_cyborg }}" }
- { name: "designate", enabled: "{{ enable_designate }}" }
- { name: "elasticsearch", enabled: "{{ enable_elasticsearch }}" }
- { name: "etcd", enabled: "{{ enable_etcd }}" }
- { name: "fluentd", enabled: "yes" }
- { name: "freezer", enabled: "{{ enable_freezer }}" }
- { name: "glance", enabled: "{{ enable_glance }}" }
- { name: "glance-tls-proxy", enabled: "{{ enable_glance }}" }
- { name: "global", enabled: "yes" }
- { name: "gnocchi", enabled: "{{ enable_gnocchi }}" }
- { name: "grafana", enabled: "{{ enable_grafana }}" }
- { name: "haproxy", enabled: "{{ enable_haproxy }}" }
- { name: "heat", enabled: "{{ enable_heat }}" }
- { name: "horizon", enabled: "{{ enable_horizon }}" }
- { name: "influxdb", enabled: "{{ enable_influxdb }}" }
- { name: "ironic", enabled: "{{ enable_ironic }}" }
- { name: "ironic-inspector", enabled: "{{ enable_ironic }}" }
- { name: "iscsid", enabled: "{{ enable_iscsid }}" }
- { name: "kafka", enabled: "{{ enable_kafka }}" }
- { name: "karbor", enabled: "{{ enable_karbor }}" }
- { name: "keepalived", enabled: "{{ enable_haproxy }}" }
- { name: "keystone", enabled: "{{ enable_keystone }}" }
- { name: "kibana", enabled: "{{ enable_kibana }}" }
- { name: "kuryr", enabled: "{{ enable_kuryr }}" }
- { name: "magnum", enabled: "{{ enable_magnum }}" }
- { name: "manila", enabled: "{{ enable_manila }}" }
- { name: "mariadb", enabled: "{{ enable_mariadb }}" }
- { name: "masakari", enabled: "{{ enable_masakari }}" }
- { name: "mistral", enabled: "{{ enable_mistral }}" }
- { name: "monasca", enabled: "{{ enable_monasca }}" }
- { name: "murano", enabled: "{{ enable_murano }}" }
- { name: "neutron", enabled: "{{ enable_neutron }}" }
- { name: "nova", enabled: "{{ enable_nova }}" }
- { name: "octavia", enabled: "{{ enable_octavia }}" }
- { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq }}" }
- { name: "panko", enabled: "{{ enable_panko }}" }
- { name: "qinling", enabled: "{{ enable_qinling }}" }
- { name: "rabbitmq", enabled: "{{ enable_rabbitmq }}" }
- { name: "rally", enabled: "{{ enable_rally }}" }
- { name: "sahara", enabled: "{{ enable_sahara }}" }
- { name: "searchlight", enabled: "{{ enable_searchlight }}" }
- { name: "senlin", enabled: "{{ enable_senlin }}" }
- { name: "skydive", enabled: "{{ enable_skydive }}" }
- { name: "solum", enabled: "{{ enable_solum }}" }
- { name: "storm", enabled: "{{ enable_storm }}" }
- { name: "swift", enabled: "{{ enable_swift }}" }
- { name: "tacker", enabled: "{{ enable_tacker }}" }
- { name: "tempest", enabled: "{{ enable_tempest }}" }
- { name: "trove", enabled: "{{ enable_trove }}" }
- { name: "vitrage", enabled: "{{ enable_vitrage }}" }
- { name: "watcher", enabled: "{{ enable_watcher }}" }
- { name: "zookeeper", enabled: "{{ enable_zookeeper }}" }
- { name: "zun", enabled: "{{ enable_zun }}" }
notify:
- Restart cron container

5
ansible/roles/common/templates/cron-logrotate-global.conf.j2
View File

@ -18,4 +18,7 @@ maxsize 100M
su root kolla
include /etc/logrotate.d
{% for service in cron_logrotate_enabled_services %}
{% include "cron-logrotate-" ~ service ~ ".conf.j2" %}
{% endfor %}

75
ansible/roles/common/templates/cron.json.j2
View File

@ -1,83 +1,12 @@
{% set cron_cmd = 'cron -f' if kolla_base_distro in ['ubuntu', 'debian'] else 'crond -s -n' %}
{% set services = [
( 'ansible', 'yes' ),
( 'aodh', enable_aodh ),
( 'barbican', enable_barbican ),
( 'blazar', enable_blazar ),
( 'ceilometer', enable_ceilometer ),
( 'chrony', enable_chrony ),
( 'cinder', enable_cinder ),
( 'cloudkitty', enable_cloudkitty ),
( 'collectd', enable_collectd ),
( 'cyborg', enable_cyborg ),
( 'designate', enable_designate ),
( 'elasticsearch', enable_elasticsearch ),
( 'etcd', enable_etcd ),
( 'freezer', enable_freezer ),
( 'glance', enable_glance ),
( 'glance-tls-proxy', enable_glance ),
( 'gnocchi', enable_gnocchi ),
( 'grafana', enable_grafana ),
( 'haproxy', enable_haproxy ),
( 'heat', enable_heat ),
( 'horizon', enable_horizon ),
( 'influxdb', enable_influxdb ),
( 'ironic', enable_ironic ),
( 'ironic-inspector', enable_ironic ),
( 'iscsid', enable_iscsid ),
( 'kafka', enable_kafka ),
( 'karbor', enable_karbor ),
( 'keepalived', enable_haproxy ),
( 'keystone', enable_keystone ),
( 'kibana', enable_kibana ),
( 'kuryr', enable_kuryr ),
( 'magnum', enable_magnum ),
( 'manila', enable_manila ),
( 'mariadb', enable_mariadb ),
( 'masakari', enable_masakari ),
( 'mistral', enable_mistral ),
( 'monasca', enable_monasca ),
( 'murano', enable_murano ),
( 'neutron', enable_neutron ),
( 'nova', enable_nova ),
( 'octavia', enable_octavia ),
( 'outward-rabbitmq', enable_outward_rabbitmq ),
( 'panko', enable_panko ),
( 'qinling', enable_qinling ),
( 'rabbitmq', enable_rabbitmq ),
( 'rally', enable_rally ),
( 'sahara', enable_sahara ),
( 'searchlight', enable_searchlight ),
( 'senlin', enable_senlin ),
( 'skydive', enable_skydive ),
( 'solum', enable_solum ),
( 'storm', enable_storm ),
( 'swift', enable_swift ),
( 'tacker', enable_tacker ),
( 'tempest', enable_tempest ),
( 'trove', enable_trove ),
( 'vitrage', enable_vitrage ),
( 'watcher', enable_watcher ),
( 'zookeeper', enable_zookeeper ),
( 'zun', enable_zun )
] %}
{
"command": "{{ cron_cmd }}",
"config_files": [
{
"source": "{{ container_config_directory }}/logrotate/global.conf",
"source": "{{ container_config_directory }}/logrotate.conf",
"dest": "/etc/logrotate.conf",
"owner": "root",
"perm": "0600"
},
{% for service, enabled in services if enabled | bool %}
{
"source": "{{ container_config_directory }}/logrotate/{{ service }}.conf",
"dest": "/etc/logrotate.d/{{ service }}.conf",
"owner": "root",
"perm": "0600"
}{{ ',' if not loop.last else '' }}
{% endfor %}
}
]
}

5
releasenotes/notes/logrotate-single-config-663d6bf154218380.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
Improves performance of the ``common`` role by generating all logrotate
configuration in a single file.