Fix ceph pool creation failure

During `openstack overcloud deploy`, ceph create pool operation
fails on RHSCSv6 as the podman command generated by 'ceph_pool'
ansible module has the volume argument as /etc/ceph.

This patch ensures to use the '/var/lib/ceph/$FSID/config/'
in the podman command generated by all the ansible moduels.

Even the 'tripleo_cephadm_ceph_cli' is updated to use the
new path.

Resolves: rhbz#2172582
Change-Id: I570ce859c3033d0f8cc36064f851419fca1a9d66
(cherry picked from commit 0c3a54a6ef)

tripleo_ansible/ansible_plugins/module_utils/ca_common.py

@@ -52,12 +52,20 @@ def container_exec(binary, container_image, spec_path=None, interactive=False):
     container_binary = os.getenv('CEPH_CONTAINER_BINARY')
     command_exec = [container_binary, 'run']
 
+    fsid = ''
     if interactive:
         command_exec.extend(['--interactive'])
 
+    if 'CEPH_FSID' in os.environ:
+        fsid = os.getenv('CEPH_FSID')
+    ceph_config_path = '/etc/ceph'
+    if fsid:
+        path = '/var/lib/ceph/{}/config'.format(fsid)
+        if os.path.exists(path):
+            ceph_config_path = path
     command_exec.extend(['--rm',
                          '--net=host',
-                         '-v', '/etc/ceph:/etc/ceph:z',
+                         '-v', '{}:/etc/ceph:z'.format(ceph_config_path),
                          '-v', '/var/lib/ceph/:/var/lib/ceph/:z',
                          '-v', '/var/log/ceph/:/var/log/ceph/:z'])
 

tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml

@@ -5,6 +5,7 @@ tripleo_cephadm_ssh_user: ceph-admin
 tripleo_cephadm_bin: /usr/sbin/cephadm
 tripleo_cephadm_cluster: ceph
 tripleo_cephadm_config_home: /etc/ceph
+tripleo_cephadm_config_home_container: "/var/lib/ceph/{{ tripleo_cephadm_fsid }}/config/"
 tripleo_cephadm_verbose: false
 tripleo_cephadm_container_ns: "quay.io/ceph"
 tripleo_cephadm_container_image: "ceph"

tripleo_ansible/roles/tripleo_cephadm/molecule/default/tasks/verify.yml

@@ -67,7 +67,7 @@
       - tripleo_cephadm_ceph_cli | regex_search('-c ' + tripleo_cephadm_conf)
       - tripleo_cephadm_ceph_cli | regex_search('-k ' + tripleo_cephadm_admin_keyring)
   vars:
-    ceph_vol: "--volume {{ tripleo_cephadm_config_home }}:{{ tripleo_cephadm_config_home }}:z"
+    ceph_vol: "--volume {{ tripleo_cephadm_config_home_container }}:{{ tripleo_cephadm_config_home }}:z"
     spec_vol: "--volume {{ tripleo_cephadm_spec }}:{{ tripleo_cephadm_container_spec }}:z"
     image: "{{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }}"
 

tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml

@@ -24,7 +24,7 @@
       {% if sensitive_data|default(false) %}
       --interactive
       {% endif %}
-      --volume {{ tripleo_cephadm_config_home }}:/etc/ceph:z
+      --volume {{ tripleo_cephadm_config_home_container }}:/etc/ceph:z
       {% if not external_cluster|default(false) -%}
       --volume {{ tripleo_cephadm_assimilate_conf }}:{{ tripleo_cephadm_assimilate_conf_container }}:z
       {% endif %}

tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_nfs.yaml

@@ -44,6 +44,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
 
 - name: Create the ingress Daemon spec definition for nfs
   when: tripleo_cephadm_ingress | default(true)
@@ -63,3 +64,4 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"

tripleo_ansible/roles/tripleo_cephadm/tasks/crush_rules.yaml

@@ -33,6 +33,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
   with_items: "{{ tripleo_cephadm_crush_rules | unique }}"
   run_once: true
 
@@ -44,6 +45,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
   register: info_ceph_default_crush_rule
   with_items: "{{ tripleo_cephadm_crush_rules | unique }}"
   run_once: true

tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/dashboard.yaml

@@ -78,6 +78,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
 
 - name: Configure Monitoring Stack
   become: true

tripleo_ansible/roles/tripleo_cephadm/tasks/export.yaml

@@ -65,6 +65,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
   register: tripleo_cephadm_client_keys
   become: true
   loop: "{{ tripleo_cephadm_keys }}"

tripleo_ansible/roles/tripleo_cephadm/tasks/keys.yaml

@@ -17,7 +17,7 @@
 # Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/
 #   https://github.com/ceph/ceph-ansible/blob/master/library/ceph_key.py
 
-- name: create cephx key(s)
+- name: Create cephx key(s)
   ceph_key:
     import_key: true
     name: "{{ item.name }}"
@@ -25,14 +25,37 @@
     mode: "{{ item.mode }}"
     secret: "{{ item.key | default('') }}"
     cluster: "{{ tripleo_cephadm_cluster }}"
-    dest: "{{ tripleo_cephadm_config_home }}"
+    dest: "{{ tripleo_cephadm_config_home_container }}"
     owner: "{{ tripleo_cephadm_uid  }}"
     group: "{{ tripleo_cephadm_uid  }}"
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
   become: true
   loop: "{{ tripleo_cephadm_keys }}"
   when:
     - tripleo_cephadm_keys is defined
     - tripleo_cephadm_keys | length > 0
+
+- name: Find the generated key(s)
+  ansible.builtin.find:
+    paths: "{{ tripleo_cephadm_config_home_container }}"
+    patterns: "*.{{ item.name }}.*"
+  register: keyring_files
+  become: true
+  loop: "{{ tripleo_cephadm_keys }}"
+  when:
+    - tripleo_cephadm_keys is defined
+    - tripleo_cephadm_keys | length > 0
+
+- name: Copy key(s) to /etc/ceph
+  ansible.builtin.copy:
+    remote_src: true
+    src: "{{ item.path }}"
+    dest: "{{ tripleo_cephadm_config_home }}"
+    owner: "{{ tripleo_cephadm_uid }}"
+    group: "{{ tripleo_cephadm_uid }}"
+    mode: '0644'
+  become: true
+  with_items: "{{ keyring_files.results | map(attribute='files') | list }}"

tripleo_ansible/roles/tripleo_cephadm/tasks/mds.yaml

@@ -42,6 +42,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
 
 - name: create filesystem pools
   ceph_pool:
@@ -59,6 +60,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
   with_items: "{{ cephfs_pools }}"
   become: true
   vars:
@@ -76,3 +78,4 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"

tripleo_ansible/roles/tripleo_cephadm/tasks/monitoring.yaml

@@ -28,6 +28,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
   when: tripleo_cephadm_dashboard_enabled | default(false) | bool or
         tripleo_cephadm_observability_data_only | default(false) | bool
 
@@ -80,6 +81,7 @@
       environment:
         CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
         CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+        CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
       with_items:
         - {"daemon": "grafana", "port": "{{ tripleo_cephadm_grafana_port | default(3100) }}"}
         - {"daemon": "prometheus", "port": "{{ tripleo_cephadm_prometheus_port | default(9092) }}"}

tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml

@@ -54,6 +54,7 @@
       environment:
         CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
         CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+        CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
       register: _rgw_keys_int
       become: true
       with_items:

tripleo_ansible/roles/tripleo_cephadm/tasks/pools.yaml

@@ -34,6 +34,7 @@
   environment:
     CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
     CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
   become: true
   with_items: "{{ tripleo_cephadm_pools }}"
   when:

tripleo_ansible/roles/tripleo_cephadm/tasks/rbd_mirror.yaml

@@ -41,6 +41,7 @@
       environment:
         CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
         CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+        CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
   when:
     - tripleo_enabled_services | intersect(['ceph_rbdmirror'])
 

tripleo_ansible/roles/tripleo_cephadm/tasks/rgw.yaml

@@ -56,6 +56,7 @@
       environment:
         CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
         CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+        CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
       vars:
         rgw_frontend_cert: "{{ slurp_cert.get('content', '') | b64decode }}"
   when: