Deploy files for multiple ceph clusters

Deploy necessary configs and keyrings for multiple
ceph cluters.  Specifically, the intent is to enable
multiple backends for cinder that can be accessed by
compute nodes.

This change will allow automatic retrieval of
ceph.conf and client keyrings from multiple ceph clusters.
Additionally, libvirt ceph client secrets will be created
to support attaching volumes to instances from multiple
ceph clusters.

Change-Id: Icee061b35f374955154a3dd703444b94da0117da

defaults/main.yml

@@ -98,3 +98,9 @@ ceph_glance_service_names:
 ceph_gnocchi_service_names:
   - gnocchi-api
   - gnocchi-metricd
+
+ceph_extra_auth_groups: "{{ ceph_extra_config_groups }}"
+ceph_extra_config_groups:
+  - cinder_backup
+  - cinder_volume
+ceph_extra_compute_group: nova_compute

doc/source/configure-ceph.rst

@@ -89,6 +89,46 @@ multiple Ceph cluster backends via the ``ceph_extra_confs`` variable.
 
 These config file sources must be present on the deployment host.
 
+Alternatively, deployers can specify more options in ``ceph_extra_confs``
+to deploy keyrings, ceph.conf files, and configure libvirt secrets.
+
+.. code-block:: console
+
+    ceph_extra_confs:
+    -  src: "/etc/openstack_deploy/ceph2.conf"
+       dest: "/etc/ceph/ceph2.conf"
+       mon_host: 192.168.1.2
+       client_name: cinder2
+       keyring_src: /etc/openstack_deploy/ceph2.client.cinder2.keyring
+       keyring_dest: /etc/ceph/ceph2.client.cinder2.keyring
+       secret_uuid: '{{ cinder_ceph_client_uuid2 }}'
+    -  src: "/etc/openstack_deploy/ceph3.conf"
+       dest: "/etc/ceph/ceph3.conf"
+       mon_host: 192.168.1.3
+       client_name: cinder3
+       keyring_src: /etc/openstack_deploy/ceph3.client.cinder3.keyring
+       keyring_dest: /etc/ceph/ceph3.client.cinder3.keyring
+       secret_uuid: '{{ cinder_ceph_client_uuid3 }}'
+
+The primary aim of this feature is to deploy multiple ceph clusters as
+cinder backends and enable nova/libvirt to mount block volumes from those
+backends.  These settings do not override the normal deployment of
+ceph client and associated setup tasks.
+
+Deploying multiple ceph clusters as cinder backends requires the following
+adjustments to each backend in ``cinder_backends``
+
+.. code-block:: console
+
+    rbd_ceph_conf: /etc/ceph/ceph2.conf
+    rbd_pool: cinder_volumes_2
+    rbd_user: cinder2
+    rbd_secret_uuid: '{{ cinder_ceph_client_uuid2 }}'
+    volume_backend_name: volumes2
+
+The dictionary keys ``rbd_ceph_conf``, ``rbd_user``, and ``rbd_secret_uuid``
+must be unique for each ceph cluster to used as a cinder_backend.
+
 Monitors
 ~~~~~~~~
 

releasenotes/notes/extra-ceph-clusters-00ad154ffb0589a6.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - Variable ``ceph_extra_confs`` has been expanded to support
+    retrieving additional ceph.conf and keyrings from multiple
+    ceph clusters automatically.
+  - Additional libvirt ceph client secrets can be defined to
+    support attaching volumes from different ceph clusters.

tasks/ceph_all.yml

@@ -60,4 +60,3 @@
   static: no
   when: cephx | bool
   tags:     ceph-auth
-

tasks/ceph_auth.yml

@@ -104,6 +104,9 @@
     src: secret.xml.j2
     dest: /tmp/nova-secret.xml
     mode: "0600"
+  with_items:
+    - secret_uuid: "{{ nova_ceph_client_uuid }}"
+      client_name: "{{ nova_ceph_client }}"
   when: inventory_hostname in groups.nova_compute and libvirt_nova_defined.rc is defined and libvirt_nova_defined.rc != 0
   tags:
     - ceph-auth-nova-libvirt-secret
@@ -149,3 +152,46 @@
   when: inventory_hostname in groups.nova_compute and libvirt_nova_set
   tags:
     - ceph-auth-nova-libvirt-secret
+
+- name: Detect correct group for extra auth
+  set_fact:
+    ceph_in_extra_auth_group: True
+  with_items: "{{ ceph_extra_auth_groups }}"
+  when:
+    - ceph_extra_confs is defined
+    - inventory_hostname in groups[item]
+  tags:
+    - ceph-auth
+    - ceph-auth-extra
+
+- include: ceph_auth_extra.yml
+  when:
+    - ceph_in_extra_auth_group is defined
+    - ceph_in_extra_auth_group | bool
+  static: no
+  tags:
+    - ceph-auth
+    - ceph-auth-extra
+
+- name: Detect extra nova uuid secret
+  set_fact:
+    ceph_extra_nova_uuid: True
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - ceph_extra_confs is defined
+    - inventory_hostname in groups[ceph_extra_compute_group]
+    - item.secret_uuid is defined
+  tags:
+    - ceph-auth
+    - ceph-auth-extra
+    - ceph-auth-libvirt-extra
+
+- include: ceph_auth_extra_compute.yml
+  when:
+    - ceph_extra_nova_uuid is defined
+    - ceph_extra_nova_uuid | bool
+  static: no
+  tags:
+    - ceph-auth
+    - ceph-auth-extra
+    - ceph-auth-libvirt-extra

tasks/ceph_auth_extra.yml

@@ -0,0 +1,77 @@
+---
+# Copyright 2016, Walmart Stores, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- name: Create keyring files for openstack clients from extra cluster(s)
+  shell: ceph auth get client.{{ item.client_name }} >/dev/null && ceph auth get-or-create client.{{ item.client_name }} > /etc/ceph/ceph.client.{{ item.client_name }}.keyring.tmp
+  with_items: ceph_extra_confs
+  delegate_to: "{{ item.mon_host }}"
+  when:
+    - item.client_name is defined
+    - item.mon_host is defined
+  with_items: ceph_extra_confs
+
+- name: Get extra keyring files
+  shell: "scp {{ item.mon_host }}:/etc/ceph/ceph.client.{{ item.client_name }}.keyring.tmp {{ item.keyring_src }}"
+  delegate_to: localhost
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.mon_host is defined
+    - item.keyring_src is defined
+    - item.client_name is defined
+  tags:
+    - ceph-config-create-config
+    - ceph-config-extra
+
+- name: Secure extra keyring file permissions
+  file:
+    path: "{{ item.keyring_src }}"
+    state: file
+    mode: 0600
+  delegate_to: localhost
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.keyring_src is defined
+
+- name: Remove temp extra keyring files
+  file:
+    path: "/etc/ceph/ceph.client.{{ item.client_name }}.keyring.tmp"
+    state: absent
+  delegate_to: "{{ item.mon_host }}"
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.mon_host is defined
+    - item.keyring_src is defined
+    - item.client_name is defined
+  tags:
+    - ceph-config-create-config
+    - ceph-config-extra
+
+- name: Create extra keyring files
+  copy:
+    src: "{{ item.keyring_src }}"
+    dest: "{{ item.keyring_dest }}"
+    owner:  root
+    group:  "{{ cephkeys_access_group }}"
+    mode:   0640
+  notify:
+    - Restart os services
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.keyring_src is defined
+    - item.keyring_dest is defined
+  tags:
+    - ceph-config-create-config
+    - ceph-config-extra

tasks/ceph_auth_extra_compute.yml

@@ -0,0 +1,166 @@
+---
+# Copyright 2016, Walmart Stores, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- name: Create key files for nova_compute on extra cluster(s)
+  shell: ceph auth get-key client.{{ item.client_name }} > /etc/ceph/ceph.client.{{ item.client_name }}.key.tmp
+  with_items: "{{ ceph_extra_confs }}"
+  delegate_to: "{{ item.mon_host }}"
+  when:
+    - item.client_name is defined
+    - item.mon_host is defined
+
+- name: Get extra key files
+  shell: "scp {{ item.mon_host }}:/etc/ceph/ceph.client.{{ item.client_name }}.key.tmp /tmp/{{ item.mon_host }}{{ item.client_name }}.key.tmp"
+  delegate_to: localhost
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.mon_host is defined
+    - item.client_name is defined
+  tags:
+    - ceph-config-create-config
+    - ceph-config-extra
+
+- name: Remove temp extra key files
+  file:
+    path: "/etc/ceph/ceph.client.{{ item.client_name }}.key.tmp"
+    state: absent
+  delegate_to: "{{ item.mon_host }}"
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.mon_host is defined
+    - item.keyring_src is defined
+    - item.client_name is defined
+  tags:
+    - ceph-config-create-config
+    - ceph-config-extra
+
+- name: Provide extra xml files to create the secrets
+  template:
+    src: secret.xml.j2
+    dest: /tmp/{{ item.mon_host }}{{ item.client_name }}-secret.xml
+    mode: "0600"
+  with_items: ceph_extra_confs
+  when:
+    - item.client_name is defined
+    - item.mon_host is defined
+    - item.secret_uuid is defined
+
+- name: Check if extra secret(s) are defined in libvirt pt1
+  shell: "virsh secret-dumpxml {{ item.secret_uuid }} 2>&1 >/dev/null && touch /tmp/{{ item.secret_uuid }}.libvirt_secret_exists"
+  always_run:   true
+  failed_when:  false
+  changed_when: false
+  with_items: ceph_extra_confs
+  when:
+    - item.secret_uuid is defined
+  tags:
+    - ceph-auth-nova-libvirt-secret
+
+- name: Check if extra secret(s) are defined in libvirt pt2
+  shell: "ls /tmp | grep \\.libvirt_secret_exists | awk -F'.' '{print $1}'"
+  always_run:   true
+  failed_when:  false
+  changed_when: false
+  register: libvirt_secret_exists
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.secret_uuid is defined
+  tags:
+    - ceph-auth-nova-libvirt-secret
+
+- name: Define libvirt nova extra secret(s)
+  shell: "virsh secret-define --file /tmp/{{ item.mon_host }}{{ item.client_name }}-secret.xml"
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.client_name is defined
+    - item.mon_host is defined
+    - item.secret_uuid is defined
+    - item.secret_uuid not in libvirt_secret_exists.results[0].stdout_lines
+  notify:
+    - Restart os services
+  tags:
+    - ceph-auth-nova-libvirt-secret
+
+- name: Check if extra secret values are set in libvirt pt1
+  shell: "virsh secret-get-value {{ item.secret_uuid }} 2>&1 >/dev/null && touch /tmp/{{ item.secret_uuid }}.libvirt_secret_value_exists "
+  always_run:   true
+  failed_when:  false
+  changed_when: false
+  register: libvirt_nova_set
+  with_items: ceph_extra_confs
+  when:
+    - item.secret_uuid is defined
+  tags:
+    - ceph-auth-nova-libvirt-secret
+
+- name: Check if extra secret values are set in libvirt pt2
+  shell: "ls /tmp | grep \\.libvirt_secret_value_exists | awk -F'.' '{print $1}'"
+  always_run:   true
+  failed_when:  false
+  changed_when: false
+  register: libvirt_secret_value_exists
+  with_items: ceph_extra_confs
+  when:
+    - item.secret_uuid is defined
+  tags:
+    - ceph-auth-nova-libvirt-secret
+
+- name: Set extra secret value(s) in libvirt
+  shell: "virsh secret-set-value --secret {{ item.secret_uuid }} --base64 $(cat /tmp/{{ item.mon_host }}{{ item.client_name }}.key.tmp)"
+  with_items: ceph_extra_confs
+  when:
+    - item.client_name is defined
+    - item.mon_host is defined
+    - item.secret_uuid is defined
+    - item.secret_uuid  not in  libvirt_secret_value_exists.results[0].stdout_lines
+  notify:
+    - Restart os services
+  tags:
+    - ceph-auth-nova-libvirt-secret
+
+# Cleanup temp files
+
+- name: Remove libvirt nova secret detection file
+  file:
+    path: "/tmp/{{ item.secret_uuid }}.libvirt_secret_exists"
+    state: "absent"
+  with_items: ceph_extra_confs
+  always_run: true
+  ignore_errors: true
+
+- name: Remove libvirt nova secret value detection file
+  file:
+    path: "/tmp/{{ item.secret_uuid }}.libvirt_secret_value_exists"
+    state: "absent"
+  with_items: ceph_extra_confs
+  always_run: true
+  ignore_errors: true
+
+- name: Remove libvirt nova secret file
+  file:
+    path: "/tmp/{{ item.mon_host }}{{ item.client_name }}-secret.xml"
+    state: "absent"
+  with_items: ceph_extra_confs
+  always_run: true
+  ignore_errors: true
+
+- name: Remove libvirt key file
+  file:
+    path: "/tmp/{{ item.mon_host }}{{ item.client_name }}.key.tmp"
+    state: "absent"
+  with_items: ceph_extra_confs
+  always_run: true
+  ignore_errors: true

tasks/ceph_config.yml

@@ -59,16 +59,22 @@
   tags:
     - ceph-config-create-config
 
-- name: Create extra ceph.conf files
+- name: Detect correct group for extra config
-  copy:
+  set_fact:
-    src: "{{ item.src }}"
+    ceph_in_extra_config_group: True
-    dest: "{{ item.dest }}"
+  when:
-    owner:  root
+     - ceph_extra_confs is defined
-    group:  root
+     - inventory_hostname in groups[item]
-    mode:   0644
+  with_items: "{{ ceph_extra_config_groups }}"
-  notify:
-  - Restart os services
-  with_items: "{{ ceph_extra_confs }}"
-  when: ceph_extra_confs is defined
   tags:
-    - ceph-config-create-config
+    - ceph-config
+    - ceph-config-extra
+
+- include: ceph_config_extra.yml
+  when:
+    - ceph_in_extra_config_group is defined
+    - ceph_in_extra_config_group | bool
+  static: no
+  tags:
+    - ceph-config
+    - ceph-config-extra

tasks/ceph_config_extra.yml

@@ -0,0 +1,56 @@
+---
+# Copyright 2016, Walmart Stores, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+- name: Get extra ceph.conf files
+  shell: "scp {{ item.mon_host }}:/etc/ceph/ceph.conf {{ item.src }}"
+  delegate_to: localhost
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.mon_host is defined
+    - item.src is defined
+  tags:
+    - ceph-config-create-config
+    - ceph-config-extra
+
+- name: Create extra ceph.conf files
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner:  root
+    group:  root
+    mode:   0644
+  notify:
+    - Restart os services
+  with_items: "{{ ceph_extra_confs }}"
+  when:
+    - item.src is defined
+    - item.dest is defined
+  tags:
+    - ceph-config-create-config
+    - ceph-config-extra
+
+- name: Add keyring section to extra ceph.conf files
+  ini_file:
+    dest: "{{ item.dest }}"
+    section: "client.{{ item.client_name }}"
+    option: keyring
+    value: "{{ item.keyring_dest }}"
+  with_items:  "{{ ceph_extra_confs }}"
+  when:
+    - item.src is defined
+    - item.dest is defined
+    - item.keyring_dest is defined
+    - item.client_name is defined

templates/secret.xml.j2

@@ -1,7 +1,7 @@
 <!-- {{ ansible_managed }} -->
 <secret ephemeral='no' private='no'>
-    <uuid>{{ nova_ceph_client_uuid}}</uuid>
+    <uuid>{{ item.secret_uuid }}</uuid>
   <usage type='ceph'>
-      <name>client.{{ nova_ceph_client }} secret</name>
+      <name>client.{{ item.client_name }} secret</name>
   </usage>
 </secret>