Add support for ceph_external_multi_config

Add new parameter which may be used to trigger
additional runs of ceph-ansible where each additional
run uses extra_vars from each key in the new
parameter to configure the overcloud to use more
than one external Ceph cluster. Supported only
for the ceph-ansible client role.

Change-Id: I4fd47445f012878e595649be8371ea212548cbd8
Implements: blueprint multiple-external-ceph
Related-Bug: #1861488

tripleo_ansible/roles/tripleo_ceph_common/defaults/main.yml

@@ -132,5 +132,5 @@ local_ceph_ansible_fetch_directory_backup: "{{ playbook_dir }}/ceph-ansible/fetc
 swift_get_url: ''
 swift_put_url: ''
 ceph_ansible_skip_tags: 'package-install,with_pkg'
-ceph_ansible_environment_variables: {}
+ceph_ansible_environment_variables: []
 ceph_ansible_inherits_calling_ansible_environment: true

tripleo_ansible/roles/tripleo_ceph_run_ansible/tasks/main.yml

@@ -43,12 +43,29 @@
       - '{% if ansible_python_interpreter is defined %}-e ansible_python_interpreter={{ ansible_python_interpreter }}{% endif %}'
       - '-{%- for number in range(0, ceph_ansible_playbook_verbosity) -%}v{% endfor %}'
       - '{% if ceph_ansible_skip_tags is defined and ceph_ansible_skip_tags|length > 0%}--skip-tags {{ ceph_ansible_skip_tags }}{% endif %}'
-      - '-i'
-      - '{{ playbook_dir }}/ceph-ansible/inventory.yml'
       - '--extra-vars'
       - '@{{ playbook_dir }}/ceph-ansible/extra_vars.yml'
       - '{% if ceph_ansible_limit is defined and ceph_ansible_limit|length > 0 %}--limit {{ ceph_ansible_limit }}{% endif %}'
 
+- name: try to find ANSIBLE_LOG_PATH in ceph_ansible_environment_variables
+  set_fact:
+    calling_ansible_log_path: "{{ item | regex_replace('ANSIBLE_LOG_PATH=', '') }}"
+  when: item is match("ANSIBLE_LOG_PATH.*")
+  loop: "{{ ceph_ansible_environment_variables }}"
+
+- name: do we need to keep looking for the ANSIBLE_LOG_PATH?
+  when: calling_ansible_log_path is undefined
+  block:
+    - name: try to find ANSIBLE_LOG_PATH in calling_ansible_environment_variables
+      set_fact:
+        calling_ansible_log_path: "{{ item | regex_replace('ANSIBLE_LOG_PATH=', '') }}"
+      when: item is match("ANSIBLE_LOG_PATH.*")
+      loop: "{{ calling_ansible_environment_variables }}"
+    - name: Set ANSIBLE_LOG_PATH to default
+      when: calling_ansible_log_path is undefined
+      set_fact:
+        calling_ansible_log_path: "{{ playbook_dir }}/ceph-ansible/ceph_ansible_command.log"
+
 - name: save ceph-ansible playbook command(s) to shell script
   copy:
     dest: "{{ playbook_dir }}/ceph-ansible/ceph_ansible_command.sh"
@@ -56,28 +73,72 @@
     content: |
       #!/usr/bin/env bash
       set -e
+      echo "Running $0" >> {{ calling_ansible_log_path }}
+      {% set inv = "-i "+ playbook_dir +"/ceph-ansible/inventory.yml" %}
       {% for playbook in ceph_ansible_playbooks %}
-      echo "Running ceph-ansible playbook {{ playbook }}"
-      {{ ceph_ansible_command_list|join(' ') }} {{ playbook }} 2>&1
+      {{ ceph_ansible_command_list|join(' ') }} {{ inv }} {{ playbook }} 2>&1
       {% endfor %}
 
-- name: run ceph-ansible (immediate log at {{ playbook_dir }}/ceph-ansible/ceph_ansible_command.log)
+- when:
+    - ceph_external_multi_config is defined
+    - (ceph_external_multi_config|length) > 0
+  block:
+    - name: set default facts for ceph_scripts list
+      set_fact:
+        ceph_prefix: "{{ playbook_dir + '/ceph-ansible/external_' }}"
+        ceph_suffix: '_ceph_ansible_command.sh'
+        ceph_default: "{{ [playbook_dir + '/ceph-ansible/ceph_ansible_command.sh'] }}"
+
+    - name: save external ceph-ansible playbook command(s) to shell script(s)
+      copy:
+        dest: "{{ ceph_prefix + item + ceph_suffix }}"
+        mode: '0755'
+        content: |
+          #!/usr/bin/env bash
+          set -e
+          echo "Running $0" >> {{ calling_ansible_log_path }}
+          {% set inv = "-i " + ceph_prefix + "inventory.ini" %}
+          {% set extra = "-e @" + ceph_prefix + item + "_extra_vars.yml" %}
+          {% for playbook in ceph_ansible_playbooks %}
+          {{ ceph_ansible_command_list|join(' ') }} {{ inv }} {{ extra }} {{ playbook }} 2>&1
+          {% endfor %}
+      loop: "{{ ceph_external_multi_config | map(attribute='cluster') | list }}"
+
+    - name: add external ceph-ansible shell script(s) to list of ceph_scripts
+      set_fact:
+        ceph_scripts: "{{ ceph_scripts|default(ceph_default) + [ceph_prefix + item + ceph_suffix] }}"
+      loop: "{{ ceph_external_multi_config | map(attribute='cluster') | list }}"
+
+- name: run {{ ceph_scripts|length|default(1) }} ceph-ansible playbook(s) (immediate log at {{ calling_ansible_log_path }})
   # Needs become to be able to read the ssh private key
   become: true
-  shell: "{{ playbook_dir }}/ceph-ansible/ceph_ansible_command.sh"
+  shell: "{{ item }}"
   # We want the output chunked into bits to prevent
   # overflowing Zaqar message size
   no_log: true
   failed_when: false
   register: outputs
   tags: run_ceph_ansible
+  when: outputs.rc is undefined or outputs.rc == 0
+  loop: "{{ ceph_scripts | default([playbook_dir + '/ceph-ansible/ceph_ansible_command.sh']) }}"
+
+- name: search output of ceph-ansible run(s) non-zero return codes
+  set_fact:
+    ceph_ansible_std_out_err: "{{ item.stdout_lines | default([]) | union(item.stderr_lines | default([])) }}"
+  no_log: true
+  when:
+    - item.rc is defined
+    - item.rc != 0
+  loop: "{{ outputs.results }}"
+  tags:
+    - run_ceph_ansible
 
 - name: print ceph-ansible output in case of failure
   debug:
-    var: outputs.stdout_lines | default([]) | union(outputs.stderr_lines | default([]))
-  failed_when: outputs.rc != 0
+    var: ceph_ansible_std_out_err
   when:
-    - outputs is changed
-    - outputs.rc != 0
+    - ceph_ansible_std_out_err is defined
+  failed_when:
+    - ceph_ansible_std_out_err is defined
   tags:
     - run_ceph_ansible

tripleo_ansible/roles/tripleo_ceph_work_dir/tasks/prepare.yml

@@ -48,3 +48,36 @@
   copy:
     dest: "{{ playbook_dir }}/ceph-ansible/extra_vars.yml"
     content: "{{ ceph_ansible_extra_vars | to_nice_yaml }}"
+
+- when:
+    - ceph_external_multi_config is defined
+    - (ceph_external_multi_config|length) > 0
+  block:
+
+    - name: generate ceph-ansible extra vars foreach ceph in ceph_external_multi_config
+      copy:
+        dest: "{{ playbook_dir }}/ceph-ansible/external_{{ item.cluster }}_extra_vars.yml"
+        content: "{{ item | default('') | to_nice_yaml }}"
+      loop: "{{ ceph_external_multi_config }}"
+
+    - name: get hosts of ceph_mon group and ceph_client group from inventory
+      # Any host in the internal ceph cluster clients group and mon group
+      # should also be a client of the additional external ceph cluster(s).
+      # The clients group is sufficient if all cases are external, but if
+      # default ceph-ansible run is internal, then roles with Ceph may not
+      # include the CephClient service like the CephMon service found in the
+      # Controller role. If the CephMon service is present, CephClient is
+      # not necessary to get a key and conf file, but the additional runs
+      # for the external Ceph cluster(s) do need the key and conf file on
+      # the hosts in that role.
+      set_fact:
+        mon_client_hosts: "{{ mon_client_hosts | default([]) + [ item ] }}"
+      with_inventory_hostnames:
+        - ceph_client
+        - ceph_mon
+
+    - name: generate ceph-ansible/external_inventory.ini for ceph_external_multi_config
+      # Build a new inventory where the collected hosts are only clients
+      template:
+        src: "external_inventory.ini.j2"
+        dest: "{{ playbook_dir }}/ceph-ansible/external_inventory.ini"

tripleo_ansible/roles/tripleo_ceph_work_dir/templates/external_inventory.ini.j2

@@ -0,0 +1,17 @@
+#jinja2: trim_blocks: "true", lstrip_blocks: "true"
+# {{ ansible_managed }}
+# This inventory is only used to configure members of the ceph clients role
+
+[clients]
+{% for host in mon_client_hosts|unique %}
+{% if hostvars[host]['ansible_ssh_user'] is defined and hostvars[host]['ansible_ssh_user']|length > 0 %}
+{{ hostvars[host]['ansible_host'] }} ansible_ssh_user={{ hostvars[host]['ansible_ssh_user'] }}
+{% else %}
+{{ hostvars[host]['ansible_host'] }}
+{% endif %}
+{% endfor %}
+[mons]
+[osds]
+[rgws]
+[mdss]
+[grafana-server]