Add tripleo_ceph_distribute_keys role

The nodes having an _admin label should be able to interact with
the ceph cluster using the ceph.client.admin keyring. This label
is typically added to the mon/mgr group, but they need the admin
keyring to run any ceph command (via cephadm shell or any other
client tool).
This change just adds a new role to properly distribute the admin
keyring right after the ceph cluster is deployed, and improves the
cephadm.yaml playbook to include this action.

Change-Id: Id17f69c714dec75ec48862f6f0cf900fae5e7c30

doc/source/roles/role-tripleo_ceph_distribute_keys.rst

@@ -0,0 +1,6 @@
+===================================
+Role - tripleo_ceph_distribute_keys
+===================================
+
+.. ansibleautoplugin::
+  :role: tripleo_ansible/roles/tripleo_ceph_distribute_keys

tripleo_ansible/playbooks/cephadm.yml

@@ -87,3 +87,13 @@
       import_role:
         name: tripleo_cephadm
         tasks_from: post
+
+- name: Distribute the admin keyring
+  hosts: ceph_mon
+  tasks:
+    - name: Distribute the admin keyring
+      import_role:
+        name: tripleo_ceph_distribute_keys
+      vars:
+        tripleo_ceph_distribute_keys_config_home: "{{ tripleo_cephadm_config_home | default('/etc/ceph') }}"
+        tripleo_ceph_distribute_keys_cluster: "{{ tripleo_cephadm_cluster | default('ceph') }}"

tripleo_ansible/roles/tripleo_ceph_distribute_keys/defaults/main.yml

@@ -0,0 +1,27 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+# All variables intended for modification should be placed in this file.
+
+# All variables within this role should have a prefix of "tripleo_ceph_distribute_keys"
+tripleo_ceph_distribute_keys_debug: "{{ (ansible_verbosity | int) >= 2 | bool }}"
+tripleo_ceph_distribute_keys_hide_sensitive_logs: true
+tripleo_ceph_distribute_keys_config_home: /etc/ceph
+tripleo_ceph_distribute_keys_cluster: "ceph"
+tripleo_ceph_distribute_keys_keyring_prefix: "{{ tripleo_ceph_distribute_keys_config_home }}/{{ tripleo_ceph_distribute_keys_cluster }}.client"
+tripleo_ceph_distribute_keys_admin_keyring: "{{ tripleo_ceph_distribute_keys_keyring_prefix }}.admin.keyring"
+tripleo_ceph_distribute_keys_ceph_uid: 167

tripleo_ansible/roles/tripleo_ceph_distribute_keys/meta/main.yml

@@ -0,0 +1,42 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+galaxy_info:
+  author: OpenStack
+  description: TripleO OpenStack Role -- tripleo_ceph_distribute_keys
+  company: Red Hat
+  license: Apache-2.0
+  min_ansible_version: 2.7
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  platforms:
+    - name: CentOS
+      versions:
+        - 7
+        - 8
+
+  galaxy_tags:
+    - tripleo
+
+
+# List your role dependencies here, one per line. Be sure to remove the '[]' above,
+# if you add dependencies to this list.
+dependencies: []

tripleo_ansible/roles/tripleo_ceph_distribute_keys/molecule/default/Dockerfile

@@ -0,0 +1,36 @@
+# Molecule managed
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
+
+{% for pkg in item.easy_install | default([]) %}
+# install pip for centos where there is no python-pip rpm in default repos
+RUN easy_install {{ pkg }}
+{% endfor %}
+
+CMD ["sh", "-c", "while true; do sleep 10000; done"]

tripleo_ansible/roles/tripleo_ceph_distribute_keys/molecule/default/converge.yml

@@ -0,0 +1,26 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  roles:
+    - role: "tripleo_ceph_distribute_keys"
+      slurp_key:
+        content: "W2NsaWVudC5hZG1pbl0KCWtleSA9IEFRQmt3OUpnL0hnakh4QUFaRXBWTjRqNUxJUlpjbkJIbEFEcUdBPT0K"
+        encoding: "base64"
+        failed: false
+        source: "/etc/ceph/foo.client.admin.keyring"

tripleo_ansible/roles/tripleo_ceph_distribute_keys/molecule/default/molecule.yml

@@ -0,0 +1,49 @@
+---
+driver:
+  name: podman
+
+log: true
+
+platforms:
+  - name: ubi8
+    hostname: ubi8
+    image: ubi8/ubi-init
+    registry:
+      url: registry.access.redhat.com
+    dockerfile: Dockerfile
+    pkg_extras: python*setuptools
+    volumes:
+      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
+      - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg
+      - /etc/dnf/vars:/etc/dnf/vars
+    privileged: true
+    environment: &env
+      http_proxy: "{{ lookup('env', 'http_proxy') }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') }}"
+    ulimits: &ulimit
+      - host
+
+provisioner:
+  inventory:
+    hosts:
+      all:
+        hosts:
+          ubi8:
+            ansible_python_interpreter: /usr/bin/python3
+  name: ansible
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml
+
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - check
+    - verify
+    - destroy
+
+verifier:
+  name: testinfra

tripleo_ansible/roles/tripleo_ceph_distribute_keys/molecule/default/prepare.yml

@@ -0,0 +1,21 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  roles:
+    - role: test_deps

tripleo_ansible/roles/tripleo_ceph_distribute_keys/tasks/distribute_conf_and_keys.yaml

@@ -0,0 +1,32 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+- name: Ensure tripleo_cephadm_config_home (e.g. /etc/ceph) exists
+  file:
+    path: "{{ tripleo_ceph_distribute_keys_config_home }}"
+    state: directory
+  become: true
+
+- name: push the admin keyring to the admin nodes of the cluster
+  tags:
+    - ceph_admin_key_rsync
+  copy:
+    dest: "{{ slurp_key.source }}"
+    content: "{{ slurp_key.content | b64decode }}"
+    mode: "0600"
+    owner: "{{ tripleo_ceph_distribute_keys_ceph_uid }}"
+    group: "{{ tripleo_ceph_distribute_keys_ceph_uid }}"
+  when:
+    - inventory_hostname != bootstrap_node

tripleo_ansible/roles/tripleo_ceph_distribute_keys/tasks/main.yml

@@ -0,0 +1,49 @@
+---
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+# "tripleo_ceph_distribute_keys" will search for and load any operating system variable file
+
+# found within the "vars/" path. If no OS files are found the task will skip.
+- name: Gather variables for each operating system
+  include_vars: "{{ item }}"
+  with_first_found:
+    - skip: true
+      files:
+        - "{{ ansible_facts['distribution'] | lower }}-{{ ansible_facts['distribution_version'] | lower }}.yml"
+        - "{{ ansible_facts['distribution'] | lower }}-{{ ansible_facts['distribution_major_version'] | lower }}.yml"
+        - "{{ ansible_facts['os_family'] | lower }}-{{ ansible_facts['distribution_major_version'] | lower }}.yml"
+        - "{{ ansible_facts['distribution'] | lower }}.yml"
+        - "{{ ansible_facts['os_family'] | lower }}-{{ ansible_facts['distribution_version'].split('.')[0] }}.yml"
+        - "{{ ansible_facts['os_family'] | lower }}.yml"
+  tags:
+    - always
+
+- name: client.admin.keyring distribution
+  become: true
+  block:
+    - name: Fetch the ceph.client.admin.keyring from the bootstrap node
+      tags:
+        - ceph_fetch_admin_keyring
+      slurp:
+        src: "{{ tripleo_ceph_distribute_keys_admin_keyring }}"
+      register: slurp_key
+      delegate_to: "{{ groups['ceph_mon'][0] }}"
+      run_once: true
+    - include: distribute_conf_and_keys.yaml
+      name: Distribute the admin keyring via ansible
+  vars:
+    - bootstrap_node: "{{ groups['ceph_mon'][0] }}"

zuul.d/molecule.yaml

@@ -13,6 +13,7 @@
       - tripleo-ansible-centos-8-molecule-tripleo_bootstrap
       - tripleo-ansible-centos-8-molecule-tripleo_cellv2
       - tripleo-ansible-centos-8-molecule-tripleo_ceph_client
+      - tripleo-ansible-centos-8-molecule-tripleo_ceph_distribute_keys
       - tripleo-ansible-centos-8-molecule-tripleo_ceph_run_ansible
       - tripleo-ansible-centos-8-molecule-tripleo_cephadm
       - tripleo-ansible-centos-8-molecule-tripleo_clients_install
@@ -74,6 +75,7 @@
       - tripleo-ansible-centos-8-molecule-tripleo_bootstrap
       - tripleo-ansible-centos-8-molecule-tripleo_cellv2
       - tripleo-ansible-centos-8-molecule-tripleo_ceph_client
+      - tripleo-ansible-centos-8-molecule-tripleo_ceph_distribute_keys
       - tripleo-ansible-centos-8-molecule-tripleo_ceph_run_ansible
       - tripleo-ansible-centos-8-molecule-tripleo_cephadm
       - tripleo-ansible-centos-8-molecule-tripleo_clients_install
@@ -136,6 +138,7 @@
       - tripleo-ansible-centos-8-molecule-tripleo_bootstrap
       - tripleo-ansible-centos-8-molecule-tripleo_cellv2
       - tripleo-ansible-centos-8-molecule-tripleo_ceph_client
+      - tripleo-ansible-centos-8-molecule-tripleo_ceph_distribute_keys
       - tripleo-ansible-centos-8-molecule-tripleo_ceph_run_ansible
       - tripleo-ansible-centos-8-molecule-tripleo_cephadm
       - tripleo-ansible-centos-8-molecule-tripleo_clients_install
@@ -248,6 +251,14 @@
     vars:
       tripleo_job_ansible_args: --skip-tags=ceph_client_rsync,clean_fetch_dir
       tripleo_role_name: tripleo_ceph_client
+- job:
+    files:
+    - ^tripleo_ansible/roles/tripleo_ceph_distribute_keys/.*
+    name: tripleo-ansible-centos-8-molecule-tripleo_ceph_distribute_keys
+    parent: tripleo-ansible-centos-8-base
+    vars:
+      tripleo_job_ansible_args: --skip-tags=ceph_fetch_admin_keyring,ceph_admin_key_rsync
+      tripleo_role_name: tripleo_ceph_distribute_keys
 - job:
     files:
     - ^tripleo_ansible/roles/tripleo_ceph_.*