Cleanup files and templates using smart sources

The files and templates we carry are almost always in a state of
maintenance. The upstream services are maintaining these files and
there's really no reason we need to carry duplicate copies of them. This
change removes all of the files we expect to get from the upstream
service. while the focus of this change is to remove configuration file
maintenance burdens it also allows the role to execute faster.

  * Source installs have the configuration files within the venv at
    "<<VENV_PATH>>/etc/<<SERVICE_NAME>>". The role will now link the
    default configuration path to this directory. When the service is
    upgraded the link will move to the new venv path.
  * Distro installs package all of the required configuration files.

To maintain our current capabilities to override configuration the
role will fetch files from the disk whenever an override is provided and
then push the fetched file back to the target using `config_template`.

Change-Id: I93cb6463ca1eb93ab7f4e7a3970a7de829efaf66
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

defaults/main.yml

@@ -458,22 +458,10 @@ keystone_optional_oslomsg_amqp1_pip_packages:
 # by the py_pkgs lookup.
 keystone_role_project_group: keystone_all
 
-#: Tunable file-based overrides
-# The contents of these files, if they exist, are read from the
-# specified path on the deployment host, interpreted by the
-# template engine and copied to the target host. If they do
-# not exist then the default files will be sourced from the
-# service git repository.
-keystone_policy_default_file_path: "/etc/openstack_deploy/keystone/policy.json"
-keystone_sso_callback_file_path: "/etc/openstack_deploy/keystone/sso_callback_template.html"
-
-# If the above-mentioned files do not exist, then the defaults
-# inside the venvs will be used, but cached at this location
-# on the deployment host. Using the cache makes the re-use
-# of the files faster when deploying, but is also required in
-# order to still be able to apply the config_template override.
-keystone_config_cache_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/keystone"
-keystone_config_cache_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
+# NOTE(cloudnull): Tunable SSO callback file file-based overrides If defined,
+#                  it'll be read from the deployment host, interpreted by the
+#                  template engine and copied to the target host.
+# keystone_sso_callback_file_path: "/etc/openstack_deploy/keystone/sso_callback_template.html"
 
 #: Tunable var-based overrides
 # The contents of these are templated over the default files.

handlers/main.yml

@@ -54,36 +54,6 @@
     - "venv changed"
     - "Restart uWSGI"
 
-# Note (odyssey4me):
-# The policy.json file is currently read continually by the services
-# and is not only read on service start. We therefore cannot template
-# directly to the file read by the service because the new policies
-# may not be valid until the service restarts. This is particularly
-# important during a major upgrade. We therefore only put the policy
-# file in place after the service has been stopped.
-#
-- name: Check whether a custom policy file is being used
-  stat:
-    path: "/etc/keystone/policy.json-{{ keystone_venv_tag }}"
-  register: _custom_policy_file
-  listen:
-    - "venv changed"
-    - "Restart uWSGI"
-
-- name: Copy new policy file into place
-  copy:
-    src: "/etc/keystone/policy.json-{{ keystone_venv_tag }}"
-    dest: "/etc/keystone/policy.json"
-    owner: "root"
-    group: "{{ keystone_system_group_name }}"
-    mode: "0640"
-    remote_src: yes
-  when:
-    - _custom_policy_file['stat']['exists'] | bool
-  listen:
-    - "venv changed"
-    - "Restart uWSGI"
-
 - name: Start uWSGI
   service:
     name: "{{ item }}"

tasks/keystone_post_install.yml

@@ -35,71 +35,74 @@
   with_items: "{{ ansible_play_hosts }}"
   when: "inventory_hostname == ansible_play_hosts[0]"
 
-- name: Check whether user-provided configuration files are provided
-  stat:
-    path: "{{ item }}"
-  with_items:
-    - "{{ keystone_policy_default_file_path }}"
-    - "{{ keystone_sso_callback_file_path }}"
-  register: _user_provided_config_files
-  delegate_to: localhost
-
-- name: Ensure that local config cache path exists on the deploy host
-  file:
-    path: "{{ keystone_config_cache_path }}"
-    state: directory
-    owner: "{{ keystone_config_cache_path_owner }}"
-  delegate_to: localhost
-  run_once: yes
-
-- name: Retrieve default configuration files from venv
-  fetch:
-    src: "{{ _keystone_etc }}/keystone/{{ keystone_sso_callback_file_path | basename }}"
-    dest: "{{ keystone_config_cache_path }}/"
-    flat: yes
-  run_once: yes
-
 - name: Copy keystone configuration files
   config_template:
-    content: "{{ item.content | default(omit) }}"
-    src: "{{ item.src | default(omit) }}"
-    dest: "{{ item.dest }}"
+    src: "keystone.conf.j2"
+    dest: "/etc/keystone/keystone.conf"
     owner: "root"
     group: "{{ keystone_system_group_name }}"
     mode: "0640"
-    config_overrides: "{{ item.config_overrides }}"
-    config_type: "{{ item.config_type }}"
-  when:
-    - item.condition | default(True)
-  with_items:
-    - src: "keystone.conf.j2"
-      dest: "/etc/keystone/keystone.conf"
-      config_overrides: "{{ keystone_keystone_conf_overrides }}"
-      config_type: "ini"
-    - src: "{{ keystone_policy_default_file_path }}"
-      dest: "/etc/keystone/policy.json-{{ keystone_venv_tag }}"
-      config_overrides: "{{ keystone_policy_overrides }}"
-      config_type: "json"
-      condition: >-
-        {{ _user_provided_config_files['results'][0]['stat']['exists'] | bool }}
+    config_overrides: "{{ keystone_keystone_conf_overrides }}"
+    config_type: "ini"
   notify:
     - Manage LB
     - Restart uWSGI
     - Restart web server
 
-- name: Copy Keystone Federation SP SSO callback template
+- name: Implement policy.json if there are overrides configured
   copy:
-    src: >-
-        {{ (_user_provided_config_files['results'][1]['stat']['exists'] | bool) |
-           ternary(keystone_sso_callback_file_path,
-           keystone_config_cache_path ~ '/' ~ keystone_sso_callback_file_path | basename) }}
-    dest: "/etc/keystone/sso_callback_template.html"
-    owner: "{{ keystone_system_user_name }}"
-    group: "{{ keystone_system_group_name }}"
-    mode: "0644"
+    content: "{{ keystone_policy_overrides | to_nice_json }}"
+    dest: "/etc/keystone/policy.json"
   when:
-    - keystone_sp != {}
+    - keystone_policy_overrides != {}
+
+# NOTE(cloudnull): This is using "cp" instead of copy with a remote_source
+#                  because we only want to copy the original files once. and we
+#                  don't want to need multiple tasks.
+- name: Preserve original configuration file(s)
+  command: "cp {{ item.target_f }} {{ item.target_f }}.original"
+  args:
+    creates: "{{ item.target_f }}.original"
+  with_items: "{{ keystone_core_files }}"
+
+- name: Fetch override files
+  fetch:
+    src: "{{ item.target_f }}"
+    dest: "{{ item.tmp_f }}"
+    flat: yes
+  changed_when: false
+  run_once: true
+  with_items: "{{ keystone_core_files }}"
+
+- name: Copy common config
+  config_template:
+    src: "{{ item.tmp_f }}"
+    dest: "{{ item.target_f }}"
+    owner: "root"
+    group: "{{ item.group | default(keystone_system_group_name) }}"
+    mode: "0640"
+    config_overrides: "{{ item.config_overrides }}"
+    config_type: "{{ item.config_type }}"
+  with_items: "{{ keystone_core_files }}"
+  notify:
+    - Restart uWSGI
+    - Restart web server
+
+- name: Cleanup fetched temp files
+  file:
+    path: "{{ item.tmp_f }}"
+    state: absent
+  changed_when: false
+  delegate_to: localhost
+  run_once: true
+  with_items: "{{ keystone_core_files }}"
+
+- name: Copy sso callback file
+  copy:
+    src: "{{ keystone_sso_callback_file_path }}"
+    dest: "/etc/keystone/sso_callback_template.html"
+  when:
+    - keystone_sso_callback_file_path is defined
   notify:
-    - Manage LB
     - Restart uWSGI
     - Restart web server

tasks/keystone_pre_install.yml

@@ -27,10 +27,10 @@
     name: "{{ item[1] }}"
     state: "present"
     system: "yes"
+  delegate_to: "{{ item[0] }}"
   with_nested:
     - "{{ ansible_play_hosts }}"
     - "{{ keystone_system_additional_groups }}"
-  delegate_to: "{{ item[0] }}"
   when: "inventory_hostname == ansible_play_hosts[0]"
 
 - name: Remove old key file(s) if found
@@ -61,26 +61,68 @@
   with_items: "{{ ansible_play_hosts }}"
   when: "inventory_hostname == ansible_play_hosts[0]"
 
+# NOTE(cloudnull): During an upgrade the local directory may exist on a source
+#                  install. If the directory does exist it will need to be
+#                  removed. This is required on source installs because the
+#                  config directory is a link.
+- name: Source config block
+  block:
+    - name: Stat config directory
+      stat:
+        path: "/etc/keystone"
+      register: keystone_conf_dir_stat
+
+    - name: Remove the config directory
+      file:
+        path: "/etc/keystone"
+        state: absent
+      when:
+        - keystone_conf_dir_stat.stat.isdir is defined and
+          keystone_conf_dir_stat.stat.isdir
+  when:
+    - keystone_install_method == 'source'
+
 # The fernet key repository is needed on all hosts even if only running against
 # one host, so the delegation preps the directories on all hosts at once.
 - name: Create keystone dir
   file:
-    path: "{{ item[1].path }}"
-    state: directory
+    path: "{{ item[1].path | default(omit) }}"
+    src: "{{ item[1].src | default(omit) }}"
+    dest: "{{ item[1].dest | default(omit) }}"
+    state: "{{ item[1].state | default('directory') }}"
     owner: "{{ item[1].owner|default(keystone_system_user_name) }}"
     group: "{{ item[1].group|default(keystone_system_group_name) }}"
-    mode: "{{ item[1].mode|default(0755) }}"
+    mode: "{{ item[1].mode | default(omit) }}"
+    force: "{{ item[1].force | default(omit) }}"
   with_nested:
     - "{{ ansible_play_hosts }}"
-    - - { path: "/openstack", mode: "0755", owner: "root", group: "root" }
-      - { path: "/etc/keystone", mode: "0750" }
-      - { path: "{{ keystone_credential_key_repository }}", mode: "0750" }
-      - { path: "{{ keystone_ldap_domain_config_dir }}", mode: "0750" }
-      - { path: "/etc/keystone/ssl" }
-      - { path: "{{ keystone_fernet_tokens_key_repository }}", mode: "2750"}
-      - { path: "{{ keystone_system_user_home }}" }
-      - { path: "/var/www/cgi-bin", owner: root, group: root }
-      - { path: "/var/www/cgi-bin/keystone" }
-      - { path: "/etc/ansible/facts.d", owner: root, group: root }
+    - - path: "/openstack"
+        mode: "0755"
+        owner: "root"
+        group: "root"
+      - path: "{{ (keystone_install_method == 'distro') | ternary('/etc/keystone', (keystone_bin | dirname) + '/etc/keystone') }}"
+        mode: "0755"
+      # NOTE(cloudnull): The "src" path is relative. This ensures all files remain
+      #                  within the host/container confines when connecting to
+      #                  them using the connection plugin or the root filesystem.
+      - dest: "/etc/keystone"
+        src: "{{ keystone_bin | dirname | regex_replace('^/', '../') }}/etc/keystone"
+        state: "{{ (keystone_install_method == 'source') | ternary('link', 'directory') }}"
+        force: "{{ (keystone_install_method == 'source') | ternary(true, omit) }}"
+      - path: "{{ keystone_credential_key_repository }}"
+        mode: "0750"
+      - path: "{{ keystone_ldap_domain_config_dir }}"
+        mode: "0750"
+      - path: "/etc/keystone/ssl"
+      - path: "{{ keystone_fernet_tokens_key_repository }}"
+        mode: "2750"
+      - path: "{{ keystone_system_user_home }}"
+      - path: "/var/www/cgi-bin"
+        owner: root
+        group: root
+      - path: "/var/www/cgi-bin/keystone"
+      - path: "/etc/ansible/facts.d"
+        owner: root
+        group: root
   delegate_to: "{{ item[0] }}"
   when: "inventory_hostname == ansible_play_hosts[0]"

vars/distro_install.yml

@@ -38,4 +38,3 @@ keystone_package_list: |-
   {{ packages }}
 
 _keystone_bin: "/usr/bin"
-_keystone_etc: "/etc"

vars/main.yml

@@ -0,0 +1,16 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+keystone_core_files: []

vars/source_install.yml

@@ -37,5 +37,4 @@ keystone_package_list: |-
   {{ packages }}
 
 _keystone_bin: "/openstack/venvs/keystone-{{ keystone_venv_tag }}/bin"
-_keystone_etc: "{{ _keystone_bin | dirname + '/etc' }}"
 keystone_uwsgi_bin: "{{ _keystone_bin }}"