Allow user overrides for ansible-role-requirements

Currently it is only possible to override the contents of the
ansible-role-requirements file as a whole.

This patch allows a user-role-requirements.yml file to be
located in the /etc/openstack_deploy directory (or the directory defined
by OSA_CONFIG_DIR). It is only necessary to list specific overridden
repos in the user role file. The boostrap-ansible.sh script is modified to
give preference to any repos named in user-role-requirements over those
in the standard ansible-role-requirements.

This gives the following benefits:

* When a deployer needs to use a locally patched version of an ansible
role, the configuration can be held under source code control in
/etc/openstack_deploy along with the rest of the environment
configuration

* Manual adjustments to ansible-role-requirements.yml are no longer lost
when running bootstrap-ansible.sh, the modifications can be kept in
userspace.

* If any additional ansible roles are required by the deployer above the
standard set these can be listed in user-role-requirements.yml

* It is very clear which roles have been overidden as they are the only
ones listed in user-role-requirements.yml

Change-Id: I689c7cf749f6611ae3dfcb1804a700d5e456947e

releasenotes/notes/user-role-requirements-1bdf5e45423f8734.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    A new optional file /etc/openstack_deploy/user-role-requirements.yml is
+    now available for a deployer to override individual entries in the upstream
+    ansible-role-requirements file. This can be used to point to alternative repos
+    containing local fixes, or to add supplementary ansible roles that are not
+    specified in the standard ansible-role-requirements.

scripts/bootstrap-ansible.sh

@@ -24,6 +24,7 @@ export HTTPS_PROXY=${HTTPS_PROXY:-""}
 # The Ansible version used for testing
 export ANSIBLE_PACKAGE=${ANSIBLE_PACKAGE:-"ansible==2.8.2"}
 export ANSIBLE_ROLE_FILE=${ANSIBLE_ROLE_FILE:-"ansible-role-requirements.yml"}
+export USER_ROLE_FILE=${USER_ROLE_FILE:-"user-role-requirements.yml"}
 export SSH_DIR=${SSH_DIR:-"/root/.ssh"}
 export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-"noninteractive"}
 # check whether to install the ARA callback plugin
@@ -185,7 +186,7 @@ if [ -f "${ANSIBLE_ROLE_FILE}" ] && [[ -z "${SKIP_OSA_ROLE_CLONE+defined}" ]]; t
 
     pushd scripts
       /opt/ansible-runtime/bin/ansible-playbook get-ansible-role-requirements.yml \
-                       -e role_file="${ANSIBLE_ROLE_FILE}"
+                       -e role_file="${ANSIBLE_ROLE_FILE}" -e user_role_file="${USER_ROLE_FILE}"
     popd
 
     unset ANSIBLE_LIBRARY

scripts/get-ansible-role-requirements.yml

@@ -60,6 +60,26 @@
       when:
         - "lookup('env', 'ZUUL_SRC_PATH') != ''"
 
+    - name: Generate a list of user overridden roles
+      set_fact:
+        user_overridden_roles: "{{ user_roles | json_query('[*].name') }}"
+
+    - name: Generate a list of roles excluding user overridden roles
+      set_fact:
+        clone_roles: "{{ (clone_roles | default([])) + [ item ] }}"
+      when:
+        - item.scm == "git" or item.scm is undefined
+        - item.name not in user_overridden_roles
+      with_items: "{{ (zuul_roles.results | default([]) |
+                       selectattr('stat', 'defined') |
+                       rejectattr('stat.exists') |
+                       map(attribute='item') | list)
+                      | default(required_roles, True) }}"
+
+    - name: Append user overridden roles
+      set_fact:
+        clone_roles: "{{ clone_roles + user_roles }}"
+
     - name: Clone git repos (with git)
       git:
         repo: "{{ item.src }}"
@@ -69,13 +89,7 @@
         depth: "{{ item.depth | default('10') }}"
         update: true
         force: true
-      when:
-        - item.scm == "git" or item.scm is undefined
-      with_items: "{{ (zuul_roles.results | default([]) |
-                       selectattr('stat', 'defined') |
-                       rejectattr('stat.exists') |
-                       map(attribute='item') | list)
-                      | default(required_roles, True) }}"
+      with_items: "{{ clone_roles }}"
       register: git_clone
       until: git_clone is success
       retries: "{{ git_clone_retries }}"
@@ -85,5 +99,7 @@
     required_roles: "{{ lookup('file', role_file) | from_yaml }}"
     role_file: "{{ playbook_dir }}/../ansible-role-requirements.yml"
     role_path_default: '/etc/ansible/roles'
+    user_roles: "{{ lookup('file', user_role_path, errors='ignore')|default([], true) | from_yaml }}"
+    user_role_path: "{{ lookup('env', 'OSA_CONFIG_DIR')|default('/etc/openstack_deploy') ~ '/' ~ user_role_file }}"
     git_clone_retries: 2
     git_clone_retry_delay: 5