From 62d9f9c10d18fcf9e6a6b5b4039a1f3b54137c03 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Sun, 5 Aug 2018 02:13:22 -0500 Subject: [PATCH] Cleanup files and templates using smart sources The files and templates we carry are almost always in a state of maintenance. The upstream services are maintaining these files and there's really no reason we need to carry duplicate copies of them. This change removes all of the files we expect to get from the upstream service. while the focus of this change is to remove configuration file maintenance burdens it also allows the role to execute faster. * Source installs have the configuration files within the venv at "<>/etc/<>". The role will now link the default configuration path to this directory. When the service is upgraded the link will move to the new venv path. * Distro installs package all of the required configuration files. To maintain our current capabilities to override configuration the role will fetch files from the disk whenever an override is provided and then push the fetched file back to the target using `config_template`. Change-Id: I93cb6463ca1eb93ab7f4e7a3970a7de829efaf66 Signed-off-by: Kevin Carter --- defaults/main.yml | 20 ++---- handlers/main.yml | 30 --------- tasks/keystone_post_install.yml | 109 ++++++++++++++++---------------- tasks/keystone_pre_install.yml | 70 ++++++++++++++++---- vars/distro_install.yml | 1 - vars/main.yml | 16 +++++ vars/source_install.yml | 1 - 7 files changed, 132 insertions(+), 115 deletions(-) create mode 100644 vars/main.yml diff --git a/defaults/main.yml b/defaults/main.yml index ada4ec2c..852c0093 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -458,22 +458,10 @@ keystone_optional_oslomsg_amqp1_pip_packages: # by the py_pkgs lookup. keystone_role_project_group: keystone_all -#: Tunable file-based overrides -# The contents of these files, if they exist, are read from the -# specified path on the deployment host, interpreted by the -# template engine and copied to the target host. If they do -# not exist then the default files will be sourced from the -# service git repository. -keystone_policy_default_file_path: "/etc/openstack_deploy/keystone/policy.json" -keystone_sso_callback_file_path: "/etc/openstack_deploy/keystone/sso_callback_template.html" - -# If the above-mentioned files do not exist, then the defaults -# inside the venvs will be used, but cached at this location -# on the deployment host. Using the cache makes the re-use -# of the files faster when deploying, but is also required in -# order to still be able to apply the config_template override. -keystone_config_cache_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/keystone" -keystone_config_cache_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}" +# NOTE(cloudnull): Tunable SSO callback file file-based overrides If defined, +# it'll be read from the deployment host, interpreted by the +# template engine and copied to the target host. +# keystone_sso_callback_file_path: "/etc/openstack_deploy/keystone/sso_callback_template.html" #: Tunable var-based overrides # The contents of these are templated over the default files. diff --git a/handlers/main.yml b/handlers/main.yml index 92a75cac..c988fed3 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -54,36 +54,6 @@ - "venv changed" - "Restart uWSGI" -# Note (odyssey4me): -# The policy.json file is currently read continually by the services -# and is not only read on service start. We therefore cannot template -# directly to the file read by the service because the new policies -# may not be valid until the service restarts. This is particularly -# important during a major upgrade. We therefore only put the policy -# file in place after the service has been stopped. -# -- name: Check whether a custom policy file is being used - stat: - path: "/etc/keystone/policy.json-{{ keystone_venv_tag }}" - register: _custom_policy_file - listen: - - "venv changed" - - "Restart uWSGI" - -- name: Copy new policy file into place - copy: - src: "/etc/keystone/policy.json-{{ keystone_venv_tag }}" - dest: "/etc/keystone/policy.json" - owner: "root" - group: "{{ keystone_system_group_name }}" - mode: "0640" - remote_src: yes - when: - - _custom_policy_file['stat']['exists'] | bool - listen: - - "venv changed" - - "Restart uWSGI" - - name: Start uWSGI service: name: "{{ item }}" diff --git a/tasks/keystone_post_install.yml b/tasks/keystone_post_install.yml index fd8cb0bd..da10d0b0 100644 --- a/tasks/keystone_post_install.yml +++ b/tasks/keystone_post_install.yml @@ -35,71 +35,74 @@ with_items: "{{ ansible_play_hosts }}" when: "inventory_hostname == ansible_play_hosts[0]" -- name: Check whether user-provided configuration files are provided - stat: - path: "{{ item }}" - with_items: - - "{{ keystone_policy_default_file_path }}" - - "{{ keystone_sso_callback_file_path }}" - register: _user_provided_config_files - delegate_to: localhost - -- name: Ensure that local config cache path exists on the deploy host - file: - path: "{{ keystone_config_cache_path }}" - state: directory - owner: "{{ keystone_config_cache_path_owner }}" - delegate_to: localhost - run_once: yes - -- name: Retrieve default configuration files from venv - fetch: - src: "{{ _keystone_etc }}/keystone/{{ keystone_sso_callback_file_path | basename }}" - dest: "{{ keystone_config_cache_path }}/" - flat: yes - run_once: yes - - name: Copy keystone configuration files config_template: - content: "{{ item.content | default(omit) }}" - src: "{{ item.src | default(omit) }}" - dest: "{{ item.dest }}" + src: "keystone.conf.j2" + dest: "/etc/keystone/keystone.conf" owner: "root" group: "{{ keystone_system_group_name }}" mode: "0640" - config_overrides: "{{ item.config_overrides }}" - config_type: "{{ item.config_type }}" - when: - - item.condition | default(True) - with_items: - - src: "keystone.conf.j2" - dest: "/etc/keystone/keystone.conf" - config_overrides: "{{ keystone_keystone_conf_overrides }}" - config_type: "ini" - - src: "{{ keystone_policy_default_file_path }}" - dest: "/etc/keystone/policy.json-{{ keystone_venv_tag }}" - config_overrides: "{{ keystone_policy_overrides }}" - config_type: "json" - condition: >- - {{ _user_provided_config_files['results'][0]['stat']['exists'] | bool }} + config_overrides: "{{ keystone_keystone_conf_overrides }}" + config_type: "ini" notify: - Manage LB - Restart uWSGI - Restart web server -- name: Copy Keystone Federation SP SSO callback template +- name: Implement policy.json if there are overrides configured copy: - src: >- - {{ (_user_provided_config_files['results'][1]['stat']['exists'] | bool) | - ternary(keystone_sso_callback_file_path, - keystone_config_cache_path ~ '/' ~ keystone_sso_callback_file_path | basename) }} - dest: "/etc/keystone/sso_callback_template.html" - owner: "{{ keystone_system_user_name }}" - group: "{{ keystone_system_group_name }}" - mode: "0644" + content: "{{ keystone_policy_overrides | to_nice_json }}" + dest: "/etc/keystone/policy.json" when: - - keystone_sp != {} + - keystone_policy_overrides != {} + +# NOTE(cloudnull): This is using "cp" instead of copy with a remote_source +# because we only want to copy the original files once. and we +# don't want to need multiple tasks. +- name: Preserve original configuration file(s) + command: "cp {{ item.target_f }} {{ item.target_f }}.original" + args: + creates: "{{ item.target_f }}.original" + with_items: "{{ keystone_core_files }}" + +- name: Fetch override files + fetch: + src: "{{ item.target_f }}" + dest: "{{ item.tmp_f }}" + flat: yes + changed_when: false + run_once: true + with_items: "{{ keystone_core_files }}" + +- name: Copy common config + config_template: + src: "{{ item.tmp_f }}" + dest: "{{ item.target_f }}" + owner: "root" + group: "{{ item.group | default(keystone_system_group_name) }}" + mode: "0640" + config_overrides: "{{ item.config_overrides }}" + config_type: "{{ item.config_type }}" + with_items: "{{ keystone_core_files }}" + notify: + - Restart uWSGI + - Restart web server + +- name: Cleanup fetched temp files + file: + path: "{{ item.tmp_f }}" + state: absent + changed_when: false + delegate_to: localhost + run_once: true + with_items: "{{ keystone_core_files }}" + +- name: Copy sso callback file + copy: + src: "{{ keystone_sso_callback_file_path }}" + dest: "/etc/keystone/sso_callback_template.html" + when: + - keystone_sso_callback_file_path is defined notify: - - Manage LB - Restart uWSGI - Restart web server diff --git a/tasks/keystone_pre_install.yml b/tasks/keystone_pre_install.yml index c2f765b8..283f52fd 100644 --- a/tasks/keystone_pre_install.yml +++ b/tasks/keystone_pre_install.yml @@ -27,10 +27,10 @@ name: "{{ item[1] }}" state: "present" system: "yes" + delegate_to: "{{ item[0] }}" with_nested: - "{{ ansible_play_hosts }}" - "{{ keystone_system_additional_groups }}" - delegate_to: "{{ item[0] }}" when: "inventory_hostname == ansible_play_hosts[0]" - name: Remove old key file(s) if found @@ -61,26 +61,68 @@ with_items: "{{ ansible_play_hosts }}" when: "inventory_hostname == ansible_play_hosts[0]" +# NOTE(cloudnull): During an upgrade the local directory may exist on a source +# install. If the directory does exist it will need to be +# removed. This is required on source installs because the +# config directory is a link. +- name: Source config block + block: + - name: Stat config directory + stat: + path: "/etc/keystone" + register: keystone_conf_dir_stat + + - name: Remove the config directory + file: + path: "/etc/keystone" + state: absent + when: + - keystone_conf_dir_stat.stat.isdir is defined and + keystone_conf_dir_stat.stat.isdir + when: + - keystone_install_method == 'source' + # The fernet key repository is needed on all hosts even if only running against # one host, so the delegation preps the directories on all hosts at once. - name: Create keystone dir file: - path: "{{ item[1].path }}" - state: directory + path: "{{ item[1].path | default(omit) }}" + src: "{{ item[1].src | default(omit) }}" + dest: "{{ item[1].dest | default(omit) }}" + state: "{{ item[1].state | default('directory') }}" owner: "{{ item[1].owner|default(keystone_system_user_name) }}" group: "{{ item[1].group|default(keystone_system_group_name) }}" - mode: "{{ item[1].mode|default(0755) }}" + mode: "{{ item[1].mode | default(omit) }}" + force: "{{ item[1].force | default(omit) }}" with_nested: - "{{ ansible_play_hosts }}" - - - { path: "/openstack", mode: "0755", owner: "root", group: "root" } - - { path: "/etc/keystone", mode: "0750" } - - { path: "{{ keystone_credential_key_repository }}", mode: "0750" } - - { path: "{{ keystone_ldap_domain_config_dir }}", mode: "0750" } - - { path: "/etc/keystone/ssl" } - - { path: "{{ keystone_fernet_tokens_key_repository }}", mode: "2750"} - - { path: "{{ keystone_system_user_home }}" } - - { path: "/var/www/cgi-bin", owner: root, group: root } - - { path: "/var/www/cgi-bin/keystone" } - - { path: "/etc/ansible/facts.d", owner: root, group: root } + - - path: "/openstack" + mode: "0755" + owner: "root" + group: "root" + - path: "{{ (keystone_install_method == 'distro') | ternary('/etc/keystone', (keystone_bin | dirname) + '/etc/keystone') }}" + mode: "0755" + # NOTE(cloudnull): The "src" path is relative. This ensures all files remain + # within the host/container confines when connecting to + # them using the connection plugin or the root filesystem. + - dest: "/etc/keystone" + src: "{{ keystone_bin | dirname | regex_replace('^/', '../') }}/etc/keystone" + state: "{{ (keystone_install_method == 'source') | ternary('link', 'directory') }}" + force: "{{ (keystone_install_method == 'source') | ternary(true, omit) }}" + - path: "{{ keystone_credential_key_repository }}" + mode: "0750" + - path: "{{ keystone_ldap_domain_config_dir }}" + mode: "0750" + - path: "/etc/keystone/ssl" + - path: "{{ keystone_fernet_tokens_key_repository }}" + mode: "2750" + - path: "{{ keystone_system_user_home }}" + - path: "/var/www/cgi-bin" + owner: root + group: root + - path: "/var/www/cgi-bin/keystone" + - path: "/etc/ansible/facts.d" + owner: root + group: root delegate_to: "{{ item[0] }}" when: "inventory_hostname == ansible_play_hosts[0]" diff --git a/vars/distro_install.yml b/vars/distro_install.yml index 73d725d3..784fd5a5 100644 --- a/vars/distro_install.yml +++ b/vars/distro_install.yml @@ -38,4 +38,3 @@ keystone_package_list: |- {{ packages }} _keystone_bin: "/usr/bin" -_keystone_etc: "/etc" diff --git a/vars/main.yml b/vars/main.yml new file mode 100644 index 00000000..aeab73d4 --- /dev/null +++ b/vars/main.yml @@ -0,0 +1,16 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +keystone_core_files: [] \ No newline at end of file diff --git a/vars/source_install.yml b/vars/source_install.yml index 415daa35..c9e6d6aa 100644 --- a/vars/source_install.yml +++ b/vars/source_install.yml @@ -37,5 +37,4 @@ keystone_package_list: |- {{ packages }} _keystone_bin: "/openstack/venvs/keystone-{{ keystone_venv_tag }}/bin" -_keystone_etc: "{{ _keystone_bin | dirname + '/etc' }}" keystone_uwsgi_bin: "{{ _keystone_bin }}"