From 9601159fe7b35a5464a1749405f0027dc5ea9b1a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 17 Oct 2017 16:30:53 +0100 Subject: [PATCH 1/4] Add support for configuration of custom fluentd filters These may be placed in $KAYOBE_CONFIG_PATH/kolla/config/fluentd/filter. --- ansible/kolla-openstack.yml | 1 + ansible/roles/kolla-openstack/defaults/main.yml | 3 +++ ansible/roles/kolla-openstack/tasks/config.yml | 10 ++++++++++ doc/source/release-notes.rst | 1 + 4 files changed, 15 insertions(+) diff --git a/ansible/kolla-openstack.yml b/ansible/kolla-openstack.yml index f00445b4f..348d7748f 100644 --- a/ansible/kolla-openstack.yml +++ b/ansible/kolla-openstack.yml @@ -185,6 +185,7 @@ kolla_extra_neutron_ml2: "{{ kolla_extra_config.neutron_ml2 | default }}" kolla_extra_nova: "{{ kolla_extra_config.nova | default }}" kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}" + kolla_extra_fluentd_filter_path: "{{ kayobe_config_path }}/kolla/config/fluentd/filter" kolla_extra_fluentd_output_path: "{{ kayobe_config_path }}/kolla/config/fluentd/output" kolla_extra_glance_path: "{{ kayobe_config_path }}/kolla/config/glance" kolla_extra_neutron_path: "{{ kayobe_config_path }}/kolla/config/neutron" diff --git a/ansible/roles/kolla-openstack/defaults/main.yml b/ansible/roles/kolla-openstack/defaults/main.yml index 752699f04..6c62778a9 100644 --- a/ansible/roles/kolla-openstack/defaults/main.yml +++ b/ansible/roles/kolla-openstack/defaults/main.yml @@ -5,6 +5,9 @@ kolla_node_custom_config_path: ############################################################################### # Fluentd configuration. +# Path to extra Fluentd filter configuration files. +kolla_extra_fluentd_filter_path: + # Path to extra Fluentd output configuration files. kolla_extra_fluentd_output_path: diff --git a/ansible/roles/kolla-openstack/tasks/config.yml b/ansible/roles/kolla-openstack/tasks/config.yml index 59d7c0c61..5f7b7687a 100644 --- a/ansible/roles/kolla-openstack/tasks/config.yml +++ b/ansible/roles/kolla-openstack/tasks/config.yml @@ -5,6 +5,7 @@ state: directory mode: 0750 with_items: + - { name: fluentd/filter, enabled: "{{ kolla_extra_fluentd_filter_path != None }}" } - { name: fluentd/output, enabled: "{{ kolla_extra_fluentd_output_path != None }}" } - { name: glance, enabled: "{{ kolla_enable_glance }}" } - { name: ironic, enabled: "{{ kolla_enable_ironic }}" } @@ -41,6 +42,15 @@ - "{{ kolla_extra_glance_path }}/*" when: kolla_extra_glance_path != None +- name: Ensure extra fluentd filter configuration files exist + template: + src: "{{ item }}" + dest: "{{ kolla_node_custom_config_path }}/fluentd/filter/{{ item | basename }}" + mode: 0640 + with_fileglob: + - "{{ kolla_extra_fluentd_filter_path }}/*.conf" + when: kolla_extra_fluentd_filter_path != None + - name: Ensure extra fluentd output configuration files exist template: src: "{{ item }}" diff --git a/doc/source/release-notes.rst b/doc/source/release-notes.rst index 9ba56294d..7afa06eff 100644 --- a/doc/source/release-notes.rst +++ b/doc/source/release-notes.rst @@ -16,6 +16,7 @@ Features applying it. * Adds support for custom neutron and nova configuration files in ``$KAYOBE_CONFIG_PATH/kolla/config/[neutron,nova]``. +* Adds support for configuration of custom fluentd filters. Upgrade Notes ------------- From d440e8e3c0cb560829927fab6f4468973241d5a7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 18 Oct 2017 12:40:02 +0100 Subject: [PATCH 2/4] Rework custom kolla-ansible configuration Supports removal of custom config files in $KAYOBE_CONFIG_PATH/kolla/config/, and adds support for ironic, keystone, magnum, murano, sahara, and swift custom config. --- ansible/kolla-openstack.yml | 6 +- .../roles/kolla-openstack/defaults/main.yml | 27 ++---- .../roles/kolla-openstack/tasks/config.yml | 96 +++++++++---------- ansible/roles/kolla-openstack/vars/main.yml | 74 ++++++++++++++ 4 files changed, 130 insertions(+), 73 deletions(-) create mode 100644 ansible/roles/kolla-openstack/vars/main.yml diff --git a/ansible/kolla-openstack.yml b/ansible/kolla-openstack.yml index 348d7748f..fbf66c846 100644 --- a/ansible/kolla-openstack.yml +++ b/ansible/kolla-openstack.yml @@ -185,8 +185,4 @@ kolla_extra_neutron_ml2: "{{ kolla_extra_config.neutron_ml2 | default }}" kolla_extra_nova: "{{ kolla_extra_config.nova | default }}" kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}" - kolla_extra_fluentd_filter_path: "{{ kayobe_config_path }}/kolla/config/fluentd/filter" - kolla_extra_fluentd_output_path: "{{ kayobe_config_path }}/kolla/config/fluentd/output" - kolla_extra_glance_path: "{{ kayobe_config_path }}/kolla/config/glance" - kolla_extra_neutron_path: "{{ kayobe_config_path }}/kolla/config/neutron" - kolla_extra_nova_path: "{{ kayobe_config_path }}/kolla/config/nova" + kolla_extra_config_path: "{{ kayobe_config_path }}/kolla/config" diff --git a/ansible/roles/kolla-openstack/defaults/main.yml b/ansible/roles/kolla-openstack/defaults/main.yml index 6c62778a9..0b22805e8 100644 --- a/ansible/roles/kolla-openstack/defaults/main.yml +++ b/ansible/roles/kolla-openstack/defaults/main.yml @@ -1,16 +1,10 @@ --- +# Path to extra kolla-ansible configuration files. +kolla_extra_config_path: + # Directory where Kolla custom configuration files will be installed. kolla_node_custom_config_path: -############################################################################### -# Fluentd configuration. - -# Path to extra Fluentd filter configuration files. -kolla_extra_fluentd_filter_path: - -# Path to extra Fluentd output configuration files. -kolla_extra_fluentd_output_path: - ############################################################################### # Glance configuration. @@ -21,9 +15,6 @@ kolla_enable_glance: # glance-registry.conf. kolla_extra_glance: -# Path to extra Glance configuration files. -kolla_extra_glance_path: - ############################################################################### # Ironic configuration. @@ -258,9 +249,6 @@ kolla_extra_neutron: # Free form extra configuration to append to ml2_conf.ini. kolla_extra_neutron_ml2: -# Path to extra Neutron configuration files. -kolla_extra_neutron_path: - ############################################################################### # Nova configuration. @@ -270,9 +258,6 @@ kolla_enable_nova: # Free form extra configuration to append to nova.conf. kolla_extra_nova: -# Path to extra Nova configuration files. -kolla_extra_nova_path: - ############################################################################### # Sahara configuration. @@ -281,3 +266,9 @@ kolla_enable_sahara: # Free form extra configuration to append to sahara.conf. kolla_extra_sahara: + +############################################################################### +# Swift configuration. + +# Whether to enable swift. +kolla_enable_swift: diff --git a/ansible/roles/kolla-openstack/tasks/config.yml b/ansible/roles/kolla-openstack/tasks/config.yml index 5f7b7687a..14a87f637 100644 --- a/ansible/roles/kolla-openstack/tasks/config.yml +++ b/ansible/roles/kolla-openstack/tasks/config.yml @@ -1,17 +1,10 @@ --- - name: Ensure the Kolla OpenStack configuration directores exist file: - path: "{{ kolla_node_custom_config_path }}/{{ item.name }}" + path: "{{ item.dest }}" state: directory mode: 0750 - with_items: - - { name: fluentd/filter, enabled: "{{ kolla_extra_fluentd_filter_path != None }}" } - - { name: fluentd/output, enabled: "{{ kolla_extra_fluentd_output_path != None }}" } - - { name: glance, enabled: "{{ kolla_enable_glance }}" } - - { name: ironic, enabled: "{{ kolla_enable_ironic }}" } - - { name: neutron, enabled: "{{ kolla_enable_neutron }}" } - - { name: nova, enabled: "{{ kolla_enable_nova }}" } - - { name: swift, enabled: "{{ kolla_enable_swift }}" } + with_items: "{{ kolla_openstack_custom_config }}" when: item.enabled | bool - name: Ensure the Kolla OpenStack configuration files exist @@ -33,33 +26,6 @@ - { src: sahara.conf.j2, dest: sahara.conf, enabled: "{{ kolla_enable_sahara }}" } when: item.enabled | bool -- name: Ensure extra glance configuration files exist - template: - src: "{{ item }}" - dest: "{{ kolla_node_custom_config_path }}/glance/{{ item | basename }}" - mode: 0640 - with_fileglob: - - "{{ kolla_extra_glance_path }}/*" - when: kolla_extra_glance_path != None - -- name: Ensure extra fluentd filter configuration files exist - template: - src: "{{ item }}" - dest: "{{ kolla_node_custom_config_path }}/fluentd/filter/{{ item | basename }}" - mode: 0640 - with_fileglob: - - "{{ kolla_extra_fluentd_filter_path }}/*.conf" - when: kolla_extra_fluentd_filter_path != None - -- name: Ensure extra fluentd output configuration files exist - template: - src: "{{ item }}" - dest: "{{ kolla_node_custom_config_path }}/fluentd/output/{{ item | basename }}" - mode: 0640 - with_fileglob: - - "{{ kolla_extra_fluentd_output_path }}/*.conf" - when: kolla_extra_fluentd_output_path != None - - name: Ensure the ironic inspector kernel and ramdisk are downloaded get_url: url: "{{ item.url }}" @@ -84,20 +50,50 @@ - kolla_enable_ironic | bool - item.path != None -- name: Ensure extra neutron configuration files exist - template: - src: "{{ item }}" - dest: "{{ kolla_node_custom_config_path }}/neutron/{{ item | basename }}" - mode: 0640 - with_fileglob: - - "{{ kolla_extra_neutron_path }}/*" - when: kolla_extra_neutron_path != None +# We support a fairly flexible mechanism of dropping config file templates into +# an 'extra' config directory, and passing these through to kolla-ansible. We +# look for matching files in the source directory to template, and also remove +# any unexpected files from the destination, to support removal of files. -- name: Ensure extra nova configuration files exist +- name: Find extra configuration files + find: + path: "{{ item.src }}" + patterns: "{{ item.patterns }}" + with_items: "{{ kolla_openstack_custom_config }}" + register: find_src_result + +- name: Find previously generated extra configuration files + find: + path: "{{ item.dest }}" + patterns: "{{ item.patterns }}" + with_items: "{{ kolla_openstack_custom_config }}" + register: find_dest_result + +- name: Ensure extra configuration files exist template: - src: "{{ item }}" - dest: "{{ kolla_node_custom_config_path }}/nova/{{ item | basename }}" + src: "{{ item.1.path }}" + dest: "{{ item.0.item.dest }}/{{ item.1.path | basename }}" mode: 0640 - with_fileglob: - - "{{ kolla_extra_nova_path }}/*" - when: kolla_extra_nova_path != None + with_subelements: + - "{{ find_src_result.results }}" + - files + - skip_missing: True + when: item.0.item.enabled | bool + +- name: Ensure unnecessary extra configuration files are absent + file: + path: "{{ item.1.path }}" + state: absent + with_subelements: + - "{{ find_dest_result.results }}" + - files + - skip_missing: True + when: + - not item.0.item.enabled or + item.1.path | basename not in src_files + - item.1.path | basename not in item.0.item.ignore | default([]) + vars: + # Find the source result that corresponds to this one. + src_result: "{{ (find_src_result.results | selectattr('item', 'equalto', item.0.item) | list)[0] }}" + # Find the list of files in the source. + src_files: "{{ src_result.files | map(attribute='path') | map('basename') | list }}" diff --git a/ansible/roles/kolla-openstack/vars/main.yml b/ansible/roles/kolla-openstack/vars/main.yml new file mode 100644 index 000000000..29019a572 --- /dev/null +++ b/ansible/roles/kolla-openstack/vars/main.yml @@ -0,0 +1,74 @@ +--- +# List of custom configuration directories. +# Each item is a dict containing the following items: +# src: Path to directory containing configuration file templates. +# dest: Path to directory in which generated files will be created. +# patterns: One or more file name patterns to match. +# enabled: Whether these files should be templated. +# ignore: Optional list of files to leave in the destination, even if disabled +# or unexpected. +kolla_openstack_custom_config: + # Fluentd filters. + - src: "{{ kolla_extra_config_path }}//fluentd/filter" + dest: "{{ kolla_node_custom_config_path }}/fluentd/filter" + patterns: "*.conf" + enabled: True + # Fluentd outputs. + - src: "{{ kolla_extra_config_path }}/fluentd/output" + dest: "{{ kolla_node_custom_config_path }}/fluentd/output" + patterns: "*.conf" + enabled: True + # Glance. + - src: "{{ kolla_extra_config_path }}/glance" + dest: "{{ kolla_node_custom_config_path }}/glance" + patterns: "*" + enabled: "{{ kolla_enable_glance }}" + # Ironic. + - src: "{{ kolla_extra_config_path }}/ironic" + dest: "{{ kolla_node_custom_config_path }}/ironic" + patterns: "*" + enabled: "{{ kolla_enable_ironic }}" + ignore: + # These are templated by kayobe, so don't remove them. + - ironic-agent.initramfs + - ironic-agent.kernel + - ironic-dnsmasq.conf + - pxelinux.default + # Keystone. + - src: "{{ kolla_extra_config_path }}/keystone" + dest: "{{ kolla_node_custom_config_path }}/keystone" + patterns: "*" + enabled: True + # Magnum. + - src: "{{ kolla_extra_config_path }}/magnum" + dest: "{{ kolla_node_custom_config_path }}/magnum" + patterns: "*" + enabled: "{{ kolla_enable_magnum }}" + # Murano. + - src: "{{ kolla_extra_config_path }}/murano" + dest: "{{ kolla_node_custom_config_path }}/murano" + patterns: "*" + enabled: "{{ kolla_enable_murano }}" + # Neutron. + - src: "{{ kolla_extra_config_path }}/neutron" + dest: "{{ kolla_node_custom_config_path }}/neutron" + patterns: "*" + enabled: "{{ kolla_enable_neutron }}" + ignore: + # These are templated by kayobe, so don't remove them. + - ml2_conf.ini + # Nova. + - src: "{{ kolla_extra_config_path }}/nova" + dest: "{{ kolla_node_custom_config_path }}/nova" + patterns: "*" + enabled: "{{ kolla_enable_nova }}" + # Sahara. + - src: "{{ kolla_extra_config_path }}/sahara" + dest: "{{ kolla_node_custom_config_path }}/sahara" + patterns: "*" + enabled: "{{ kolla_enable_sahara }}" + # Swift. + - src: "{{ kolla_extra_config_path }}/swift" + dest: "{{ kolla_node_custom_config_path }}/swift" + patterns: "*" + enabled: "{{ kolla_enable_swift }}" From 7fc93906de3ab71c7970c8ae696dd3c26692fbf2 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 18 Oct 2017 17:24:29 +0100 Subject: [PATCH 3/4] Add all extra-configurable services to release notes --- doc/source/release-notes.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/release-notes.rst b/doc/source/release-notes.rst index 7afa06eff..851ac631b 100644 --- a/doc/source/release-notes.rst +++ b/doc/source/release-notes.rst @@ -14,9 +14,9 @@ Features * Adds a ``--display`` argument to ``kayobe physical network configure`` command. This will output the candidate switch configuration without applying it. -* Adds support for custom neutron and nova configuration files in - ``$KAYOBE_CONFIG_PATH/kolla/config/[neutron,nova]``. -* Adds support for configuration of custom fluentd filters. +* Adds support for configuration of custom fluentd filters, and additional + config file templates for heat, ironic, keystone, magnum, murano, sahara, and + swift in ``$KAYOBE_CONFIG_PATH/kolla/config//``. Upgrade Notes ------------- From 9187216b781d04adef0874c2fe08a88bc0688237 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 18 Oct 2017 17:25:33 +0100 Subject: [PATCH 4/4] Add extra config for heat --- ansible/roles/kolla-openstack/vars/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ansible/roles/kolla-openstack/vars/main.yml b/ansible/roles/kolla-openstack/vars/main.yml index 29019a572..60b68b628 100644 --- a/ansible/roles/kolla-openstack/vars/main.yml +++ b/ansible/roles/kolla-openstack/vars/main.yml @@ -23,6 +23,11 @@ kolla_openstack_custom_config: dest: "{{ kolla_node_custom_config_path }}/glance" patterns: "*" enabled: "{{ kolla_enable_glance }}" + # Heat. + - src: "{{ kolla_extra_config_path }}/heat" + dest: "{{ kolla_node_custom_config_path }}/heat" + patterns: "*" + enabled: "{{ kolla_enable_heat }}" # Ironic. - src: "{{ kolla_extra_config_path }}/ironic" dest: "{{ kolla_node_custom_config_path }}/ironic"