Rework cephadm key and pools creation for Quincy+

From Quincy we have issues w/ the modules used to build and import keys within the cluster. Similar issues happen when the pool module is called. This patch fixes both the pools and keys creation and relies on the common cephadm cli instead of the existing python module. Change-Id: I7c0e2c95cef09ef3bc9ca341a70afc06859e2c21
2023-05-26 21:47:35 +02:00 · 2023-05-26 21:47:35 +02:00 · d1a48ac2f2
parent 36e4da151c
commit d1a48ac2f2
18 changed files with 103 additions and 223 deletions
--- a/tripleo_ansible/playbooks/cephadm.yml
+++ b/tripleo_ansible/playbooks/cephadm.yml
@ -28,6 +28,11 @@
      when:
        - not tripleo_cephadm_deployed_ceph | bool

+    - name: Selinux
+      import_role:
+        name: tripleo_cephadm
+        tasks_from: selinux
+
    - name: Apply ceph_conf_overrides on update
      import_role:
        name: tripleo_cephadm
@ -116,3 +121,8 @@
      vars:
        tripleo_ceph_distribute_keys_config_home: "{{ tripleo_cephadm_config_home | default('/etc/ceph') }}"
        tripleo_ceph_distribute_keys_cluster: "{{ tripleo_cephadm_cluster | default('ceph') }}"
+
+    - name: Selinux
+      import_role:
+        name: tripleo_cephadm
+        tasks_from: selinux
--- a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/converge.yml
+++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/converge.yml
@ -46,7 +46,7 @@
        tasks_from: export
      vars:
        ceph_mon_dump: "{{ ceph_mon_mock_dump }}"
-        tripleo_cephadm_client_keys: "{{ mock_ceph_keys }}"
+        tripleo_cephadm_keys: "{{ mock_ceph_keys }}"

    - name: Run verify tasks
      include_tasks: tasks/verify.yml
--- a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/vars/mock_ceph_keys.yml
+++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/vars/mock_ceph_keys.yml
@ -14,96 +14,4 @@
 # License for the specific language governing permissions and limitations
 # under the License.

-mock_ceph_keys:
-  results:
-    - ansible_facts:
-        discovered_interpreter_python: /usr/libexec/platform-python
-      ansible_loop_var: item
-      changed: false
-      cmd:
-        - podman
-        - run
-        - --rm
-        - --net=host
-        - -v
-        - /etc/ceph:/etc/ceph:z
-        - -v
-        - /var/lib/ceph/:/var/lib/ceph/:z
-        - -v
-        - /var/log/ceph/:/var/log/ceph/:z
-        - --entrypoint=ceph
-        - undercloud.ctlplane.mydomain.tld:8787/ceph-ci/daemon:v5.0.7-stable-5.0-octopus-centos-8-x86_64
-        - -n
-        - client.admin
-        - -k
-        - /etc/ceph/ceph.client.admin.keyring
-        - --cluster
-        - ceph
-        - auth
-        - get
-        - client.openstack
-        - -f
-        - json
-      delta: '0:00:01.501594'
-      end: '2021-01-29 19:48:16.372821'
-      failed: false
-      invocation:
-        module_args:
-          attributes: null
-          backup: null
-          caps: null
-          cluster: ceph
-          content: null
-          delimiter: null
-          dest: /etc/ceph/
-          directory_mode: null
-          follow: false
-          force: null
-          group: null
-          import_key: true
-          mode: null
-          name: client.openstack
-          output_format: json
-          owner: null
-          regexp: null
-          remote_src: null
-          secret: null
-          selevel: null
-          serole: null
-          setype: null
-          seuser: null
-          src: null
-          state: info
-          unsafe_writes: null
-          user: client.admin
-          user_key: null
-      item:
-        caps:
-          mgr: allow *
-          mon: profile rbd
-          osd: profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images
-        key: AQATZBBgAAAAABAAUl/GZvcldk6G74AoZ2v2rg==
-        mode: '0600'
-        name: client.openstack
-      rc: 0
-      start: '2021-01-29 19:48:14.871227'
-      stderr: exported keyring for client.openstack
-      stderr_lines:
-        - exported keyring for client.openstack
-      stdout: >-
-        [{"entity":"client.openstack",
-          "key":"AQATZBBgAAAAABAAUl/GZvcldk6G74AoZ2v2rg==",
-          "caps":{
-            "mgr":"allow *",
-            "mon":"profile rbd",
-            "osd":"profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images"
-          }}]
-      stdout_lines: >-
-        - ''
-        - '[{"entity":"client.openstack",
-             "key":"AQATZBBgAAAAABAAUl/GZvcldk6G74AoZ2v2rg==",
-             "caps":{
-               "mgr":"allow *",
-               "mon":"profile rbd",
-               "osd":"profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images"
-            }}]'
+mock_ceph_keys: [{'name': 'client.openstack', 'key': 'AQATZBBgAAAAABAAUl/GZvcldk6G74AoZ2v2rg==', 'mode': '0600', 'caps': {'mgr': 'allow *', 'mon': 'profile rbd', 'osd': 'profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images'}}]
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml
@ -14,12 +14,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.

- name: Stat cephadm config home container directory
-  stat:
-    path: "{{ tripleo_cephadm_config_home_container }}"
-  register: tripleo_cephadm_config_home_container_stat
-  become: true
-
 - name: Set ceph CLI
  set_fact:
    tripleo_cephadm_ceph_cli: >-
@ -30,11 +24,7 @@
      {% if sensitive_data|default(false) %}
      --interactive
      {% endif %}
-      {% if tripleo_cephadm_config_home_container_stat.stat.exists %}
-      --volume {{ tripleo_cephadm_config_home_container }}:/etc/ceph:z
-      {% else %}
      --volume {{ tripleo_cephadm_config_home }}:/etc/ceph:z
-      {% endif %}
      {% if not external_cluster|default(false) -%}
      --volume {{ tripleo_cephadm_assimilate_conf }}:{{ tripleo_cephadm_assimilate_conf_container }}:z
      {% endif %}
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/crush_rules.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/crush_rules.yaml
@ -33,7 +33,6 @@
  environment:
    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
  with_items: "{{ tripleo_cephadm_crush_rules | unique }}"
  run_once: true

@ -45,7 +44,6 @@
  environment:
    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
  register: info_ceph_default_crush_rule
  with_items: "{{ tripleo_cephadm_crush_rules | unique }}"
  run_once: true
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/dashboard.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/dashboard.yaml
@ -78,7 +78,6 @@
  environment:
    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"

 - name: Configure Monitoring Stack
  become: true
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/export.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/export.yaml
@ -57,24 +57,6 @@
  set_fact:
    external_cluster_mon_ips: "{{ tripleo_cephadm_mons_list | join(',') }}"

- name: Extract keys
-  ceph_key:
-    name: "{{ item.name }}"
-    cluster: "{{ tripleo_cephadm_cluster }}"
-    state: info
-  environment:
-    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
-    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
-  register: tripleo_cephadm_client_keys
-  become: true
-  loop: "{{ tripleo_cephadm_keys }}"
-  when:
-    - tripleo_cephadm_keys is defined
-    - tripleo_cephadm_keys | length > 0
-  tags:
-    - cephadm_extract_keys
-
 - name: Ensure tripleo_ceph_client_vars path exists
  file:
    path: "{{ tripleo_ceph_client_vars | dirname }}"
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/keys.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/keys.yaml
@ -17,56 +17,28 @@
 # Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/
 #   https://github.com/ceph/ceph-ansible/blob/master/library/ceph_key.py

- name: Prepare tripleo_ceph_key_dest
-  set_fact:
-    tripleo_ceph_key_dest: >-
-      {%- if tripleo_cephadm_config_home_container_stat.stat.exists -%}
-      {{ tripleo_cephadm_config_home_container }}
-      {%- else -%}
-      {{ tripleo_cephadm_config_home }}
-      {%- endif -%}
-
- name: Create cephx key(s)
-  ceph_key:
-    import_key: true
-    name: "{{ item.name }}"
-    caps: "{{ item.caps }}"
-    mode: "{{ item.mode }}"
-    secret: "{{ item.key | default('') }}"
-    cluster: "{{ tripleo_cephadm_cluster }}"
-    dest: "{{ tripleo_ceph_key_dest }}"
-    owner: "{{ tripleo_cephadm_uid  }}"
-    group: "{{ tripleo_cephadm_uid  }}"
-  environment:
-    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
-    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
+- name: Render cephx key(s) in tripleo_cephadm_config_home
  become: true
-  loop: "{{ tripleo_cephadm_keys }}"
-  when:
-    - tripleo_cephadm_keys is defined
-    - tripleo_cephadm_keys | length > 0
-
- name: Find the generated key(s)
-  ansible.builtin.find:
-    paths: "{{ tripleo_cephadm_config_home_container }}"
-    patterns: "*.{{ item.name }}.*"
-  register: keyring_files
-  become: true
-  loop: "{{ tripleo_cephadm_keys }}"
-  when:
-    - tripleo_cephadm_keys is defined
-    - tripleo_cephadm_keys | length > 0
-    - tripleo_cephadm_config_home_container_stat.stat.exists | bool
-
- name: Copy key(s) to /etc/ceph
-  ansible.builtin.copy:
-    remote_src: true
-    src: "{{ item.path }}"
-    dest: "{{ tripleo_cephadm_config_home }}"
+  template:
+    src: templates/ceph_key.j2
+    dest: "{{ tripleo_cephadm_config_home | default('/etc/ceph') }}/{{ tripleo_cephadm_cluster | default('ceph') }}.{{ item.name }}.keyring"
+    mode: 0644
+    force: true
    owner: "{{ tripleo_cephadm_uid }}"
    group: "{{ tripleo_cephadm_uid }}"
-    mode: '0644'
+  loop: "{{ tripleo_cephadm_keys| default([]) }}"
+  when:
+    - tripleo_cephadm_keys is defined
+    - tripleo_cephadm_keys | length > 0
+
+- name: Get ceph_cli
+  include_tasks: ceph_cli.yaml
+
+- name: Import cephx key(s) within the Ceph cluster
+  command: "{{ tripleo_cephadm_ceph_cli }} auth import -i {{ tripleo_cephadm_config_home | default('/etc/ceph') }}/{{ tripleo_cephadm_cluster | default('ceph') }}.{{ item.name }}.keyring"
  become: true
-  with_items: "{{ keyring_files.results | map(attribute='files') | list }}"
-  when: tripleo_cephadm_config_home_container_stat.stat.exists | bool
+  loop: "{{ tripleo_cephadm_keys| default([]) }}"
+  changed_when: false
+  when:
+    - tripleo_cephadm_keys is defined
+    - tripleo_cephadm_keys | length > 0
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/mds.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/mds.yaml
@ -42,31 +42,23 @@
  environment:
    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"

- name: create filesystem pools
-  ceph_pool:
-    name: "{{ item.name }}"
-    cluster: "{{ tripleo_cephadm_cluster }}"
-    pg_num: "{{ item.pg_num | default(omit) }}"
-    pgp_num: "{{ item.pgp_num | default(omit) }}"
-    size: "{{ item.size | default(omit) }}"
-    min_size: "{{ item.min_size | default(omit) }}"
-    pool_type: "{{ item.type | default('replicated') }}"
-    rule_name: "{{ item.rule_name | default(omit) }}"
-    erasure_profile: "{{ item.erasure_profile | default(omit) }}"
-    pg_autoscale_mode: "{{ item.pg_autoscale_mode | default(omit) }}"
-    target_size_ratio: "{{ item.target_size_ratio | default(omit) }}"
-  environment:
-    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
-    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
-  with_items: "{{ cephfs_pools }}"
+- name: Create filesystem Pools
  become: true
+  block:
+    - name: Get ceph_cli
+      include_tasks: ceph_cli.yaml
+
+    - name: Create cephfs pools
+      command: "{{ tripleo_cephadm_ceph_cli }} osd pool create {{ item.name }} {{ item.pg_num | default('') }} {{ item.rule_name }} --autoscale-mode {{ item.pg_autoscale | default('on') }}"
+      loop: "{{ cephfs_pools | default([]) }}"
  vars:
    cephfs_pools:
      - "{{ cephfs_data_pool }}"
      - "{{ cephfs_metadata_pool }}"
+  when:
+    - cephfs_data_pool is defined
+    - cephfs_metadata_pool is defined

 - name: create ceph filesystem
  ceph_fs:
@ -78,4 +70,3 @@
  environment:
    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/monitoring.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/monitoring.yaml
@ -78,7 +78,6 @@
      environment:
        CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
        CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-        CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
      with_items:
        - {"daemon": "grafana", "port": "{{ tripleo_cephadm_grafana_port | default(3100) }}"}
        - {"daemon": "prometheus", "port": "{{ tripleo_cephadm_prometheus_port | default(9092) }}"}
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml
@ -54,7 +54,6 @@
      environment:
        CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
        CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-        CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
      register: _rgw_keys_int
      become: true
      with_items:
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/pools.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/pools.yaml
@ -17,26 +17,22 @@
 # Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/
 #   https://github.com/ceph/ceph-ansible/blob/master/library/ceph_pool.py

- name: Create pool(s)
-  ceph_pool:
-    name: "{{ item.name }}"
-    cluster: "{{ tripleo_cephadm_cluster }}"
-    pg_num: "{{ item.pg_num | default(omit) }}"
-    pgp_num: "{{ item.pgp_num | default(omit) }}"
-    pg_autoscale_mode: "{{ item.pg_autoscale_mode | default(omit) }}"
-    target_size_ratio: "{{ item.target_size_ratio | default(omit) }}"
-    size: "{{ item.size | default(omit) }}"
-    min_size: "{{ item.min_size | default(omit) }}"
-    pool_type: "{{ item.type | default('replicated') }}"
-    rule_name: "{{ item.rule_name | default(omit) }}"
-    erasure_profile: "{{ item.erasure_profile | default(omit) }}"
-    application: "{{ item.application | default(omit) }}"
-  environment:
-    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
-    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-    CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
+
+- name: Create Ceph Pools
  become: true
-  with_items: "{{ tripleo_cephadm_pools }}"
+  block:
+    - name: Get ceph_cli
+      include_tasks: ceph_cli.yaml
+
+    - name: Create pool
+      command: "{{ tripleo_cephadm_ceph_cli }} osd pool create {{ item.name }} {{ item.pg_num | default('') }} {{ item.rule_name }} --autoscale-mode {{ item.pg_autoscale | default('on') }}"
+      loop: "{{ tripleo_cephadm_pools | default([]) }}"
+      changed_when: false
+
+    - name: Enable application on Ceph pools
+      command: "{{ tripleo_cephadm_ceph_cli }} osd pool application enable {{ item.name }} {{ item.application }}"
+      loop: "{{ tripleo_cephadm_pools | default([]) }}"
+      changed_when: false
  when:
    - tripleo_cephadm_pools is defined
    - tripleo_cephadm_pools | length > 0
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/rbd_mirror.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/rbd_mirror.yaml
@ -41,7 +41,6 @@
      environment:
        CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
        CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-        CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
  when:
    - tripleo_enabled_services | intersect(['ceph_rbdmirror'])

--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/rgw.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/rgw.yaml
@ -56,7 +56,6 @@
      environment:
        CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
        CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
-        CEPH_FSID: "{{ tripleo_cephadm_fsid }}"
      vars:
        rgw_frontend_cert: "{{ slurp_cert.get('content', '') | b64decode }}"
  when:
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/selinux.yaml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/selinux.yaml
@ -0,0 +1,32 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+- name: Selinux context
+  block:
+    - name: Set fcontext on ceph config and admin keyring
+      become: true
+      sefcontext:
+        target: "{{ item }}"
+        setype: etc_t
+        state: present
+      with_items:
+        - "{{ tripleo_cephadm_conf }}"
+        - "{{ tripleo_cephadm_admin_keyring }}"
+      when:
+        - ansible_facts.selinux.status == "enabled"
+    - name: Restorecon on config directory
+      become: true
+      command: "restorecon -R -v {{ tripleo_cephadm_config_home }}"
--- a/tripleo_ansible/roles/tripleo_cephadm/tasks/toggle_cephadm.yml
+++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/toggle_cephadm.yml
@ -41,3 +41,6 @@
  changed_when: false
  become: true
  loop: "{{ tripleo_cephadm_toggle_cmds }}"
+
+- name: Selinux
+  include_tasks: selinux.yaml
--- a/tripleo_ansible/roles/tripleo_cephadm/templates/ceph_client.yaml.j2
+++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph_client.yaml.j2
@ -3,13 +3,11 @@ tripleo_ceph_client_fsid: {{ tripleo_cephadm_fsid }}
 tripleo_ceph_client_cluster: {{ tripleo_cephadm_cluster }}
 external_cluster_mon_ips: "{{ external_cluster_mon_ips }}"
 keys:
-{% for ceph_key_cmd in tripleo_cephadm_client_keys.results %}
-{%   for cephx in (ceph_key_cmd.stdout | from_json) %}
- name: {{ cephx.entity }}
+{% for cephx in tripleo_cephadm_keys %}
+- name: {{ cephx.name }}
  key: {{ cephx.key }}
  caps:
-{%     for key, value in cephx.caps.items() %}
+{%  for key, value in cephx.caps.items() %}
    {{ key }}: {{ value }}
-{%     endfor %}
-{%   endfor %}
+{%  endfor %}
 {% endfor %}
--- a/tripleo_ansible/roles/tripleo_cephadm/templates/ceph_key.j2
+++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph_key.j2
@ -0,0 +1,5 @@
+[{{ item.name }}]
+   key = "{{ item.key }}"
+{% for key, value in item.caps.items() %}
+   caps {{ key }} = {{ value }}
+{% endfor %}