Support multiple external ceph clusters in tripleo-ceph-client

Add DCN map variable which can override Ceph Mon IPs, FSID, Name
and keys list. This variable may used to populate the fetch dir
with more than one set of keys and conf files per Ceph cluster
before the keys/conf file are synchronized. The user may then
iterate through a list of such maps and then inclue the role
for each of those maps.

Co-Authored-By: Francesco Pantano <fpantano@redhat.com>
Implements: blueprint tripleo-ceph-client
Change-Id: I938ab604859fda88f3491399444841a3a373d162
This commit is contained in:
John Fulton 2020-12-08 21:01:49 +00:00
parent 9359639940
commit 49ea5f0194
6 changed files with 242 additions and 5 deletions

View File

@ -75,8 +75,126 @@
when: verbosity
vars:
tripleo_ceph_client_cluster: "ceph"
verbosity: false
verbosity: true
expected_files:
- '{{ tripleo_ceph_client_cluster }}.client.glance.keyring'
- '{{ tripleo_ceph_client_cluster }}.client.manila.keyring'
- '{{ tripleo_ceph_client_cluster }}.conf'
- name: Run tripleo_ceph_client with ceph_external_multi_config
block:
- include_role:
name: tripleo_ceph_client
name: Call tripleo_ceph_client and pass multiple
vars:
tripleo_ceph_client_config_home: "/var/lib/tripleo-config/ceph"
multiple: "{{ item }}"
loop: "{{ ceph_external_multi_config }}"
- name: grep expected content from generated Ceph config files for dcn0/dcn1
shell: "grep {{ item.fsid }} {{ tripleo_ceph_client_fetch_dir +'/'+ item.cluster +'.conf' }} | wc -l"
register: grep_ceph_conf_dcn
loop:
- {cluster: 'dcn0', fsid: 'b6d6c1b9-cb48-4e1f-a48b-63b27a704ab1'}
- {cluster: 'dcn1', fsid: 'cf9b830d-8c6a-418f-b6ce-a9156ae014ba'}
- name: Fail if expected FSID is not found in dcn0 or dcn1 Ceph conf file
fail:
msg: "'{{ item.cmd }}' returned {{ item.stdout }} instead of 1"
when: not item.stdout == '1'
loop: "{{ grep_ceph_conf_dcn.results }}"
- name: grep expected content from generated cephx files for dcn0/dcn1
shell: "grep {{ item.key }} {{ tripleo_ceph_client_fetch_dir +'/'+ item.cluster + '.client.openstack.keyring' }} | wc -l"
register: grep_cephx_dcn
loop:
- {cluster: 'dcn0', key: 'AQAqFN1fAAAAABAAkHoVtZxSvvRbwYtZ4LwNZA=='}
- {cluster: 'dcn1', key: 'AQAqJt1fAAAAABAATLe3ZcGWrahsG2wp17nPVQ=='}
- name: Fail if expected cephx key is not found in dcn0 or dcn1 keyring file
fail:
msg: "'{{ item.cmd }}' returned {{ item.stdout }} instead of 1"
when: not item.stdout == '1'
loop: "{{ grep_cephx_dcn.results }}"
vars:
ceph_external_multi_config:
- cluster: dcn0
ceph_conf_overrides:
client:
keyring: /etc/ceph/dcn0.client.openstack.keyring
dashboard_enabled: false
external_cluster_mon_ips: 192.168.24.7,192.168.24.14,192.168.24.24
fsid: b6d6c1b9-cb48-4e1f-a48b-63b27a704ab1
keys:
- caps:
mgr: allow *
mon: profile rbd
osd: profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images
key: AQAqFN1fAAAAABAAkHoVtZxSvvRbwYtZ4LwNZA==
mode: '0600'
name: client.openstack
- cluster: dcn1
ceph_conf_overrides:
client:
keyring: /etc/ceph/dcn1.client.openstack.keyring
dashboard_enabled: false
external_cluster_mon_ips: 192.168.25.7,192.168.25.14,192.168.25.24
fsid: cf9b830d-8c6a-418f-b6ce-a9156ae014ba
keys:
- caps:
mgr: allow *
mon: profile rbd
osd: profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images
key: AQAqJt1fAAAAABAATLe3ZcGWrahsG2wp17nPVQ==
mode: '0600'
name: client.openstack
- name: Test effective client tasks file
block:
- name: Test include case
include_tasks: ../../tasks/effective_clients_limit.yml
vars: # I am overriding this magic variable only for testing purposes
ansible_limit: 'undercloud:controller-2'
- name: Fail if include case has an unxpected value
fail:
msg: "tripleo_ceph_client_effective_clients returned
'{{ tripleo_ceph_client_effective_clients }}'
instead of ['controller-2']"
when: tripleo_ceph_client_effective_clients != ['controller-2']
- name: reset facts
set_fact:
tripleo_ceph_client_exclude: []
tripleo_ceph_client_include: []
- name: Test exclude case
include_tasks: ../../tasks/effective_clients_limit.yml
vars: # I am overriding this magic variable only for testing purposes
ansible_limit: 'undercloud:!controller-2'
- name: Fail if exclude case has an unxpected value
fail:
msg: "tripleo_ceph_client_effective_clients returned
'{{ tripleo_ceph_client_effective_clients }}' instead
of ['compute-0', 'controller-0', 'controller-1']"
when: tripleo_ceph_client_effective_clients | sort !=
['compute-0', 'controller-0', 'controller-1'] | sort
- name: reset facts
set_fact:
tripleo_ceph_client_exclude: []
tripleo_ceph_client_include: []
- name: Test include/exclude case
include_tasks: ../../tasks/effective_clients_limit.yml
vars: # I am overriding this magic variable only for testing purposes
ansible_limit: 'undercloud,!controller-2,controller-1'
- name: Fail if include/exclude case has an unxpected value
fail:
msg: "tripleo_ceph_client_effective_clients returned
'{{ tripleo_ceph_client_effective_clients }}'
instead of ['controller-1']"
when: tripleo_ceph_client_effective_clients != ['controller-1']
vars:
client_group:
- controller-0
- controller-1
- controller-2
- compute-0

View File

@ -18,9 +18,10 @@
block:
- name: Get ceph_mon_ip addresses
set_fact:
tripleo_ceph_client_mon_ips: "{{ overcloud.get('ceph_mon_node_ips', []) }}"
vars:
overcloud: "{{ lookup('file', playbook_dir + '/group_vars/overcloud.json') }}"
tripleo_ceph_client_mon_ips: "{{ (tripleo_ceph_client_mon_ips | default([]))
| union([(hostvars[item]['storage_ip']
| default(hostvars[item]['ctlplane_ip']))]) }}"
loop: "{{ groups['ceph_mon'] | list }}"
when:
- (tripleo_enabled_services | intersect(['ceph_mon'])) and
(groups['ceph_mon'] | length > 0)

View File

@ -0,0 +1,60 @@
---
# Copyright 2020 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# sanitize limit to always use ':'
# external_deploy_steps_tasks with limit must always include undercloud
# but we do not want undercloud given how we will delegate from it
- name: set limit list
set_fact:
tripleo_ceph_client_limit_list: "{{ ( ansible_limit |
regex_replace(':|,', ':') |
regex_replace('undercloud:', '') ).split(':') }}"
# 'a:!b:c' --> only 'b' (without '!')
- name: set exclude hosts
set_fact:
tripleo_ceph_client_exclude: "{{ tripleo_ceph_client_exclude|default([])
+ [ item | regex_replace('!') ] }}"
when: item is regex('^\!')
loop: "{{ tripleo_ceph_client_limit_list }}"
# 'a:!b:c' --> only 'a:c'
- name: set include hosts
set_fact:
tripleo_ceph_client_include: "{{ tripleo_ceph_client_include|default([])
+ [item] }}"
when: item is not regex('^\!')
loop: "{{ tripleo_ceph_client_limit_list }}"
- name: set effective deny list with exclude
set_fact:
tripleo_ceph_client_effective_clients: "{{ client_group |
symmetric_difference(tripleo_ceph_client_exclude) |
list }}"
when:
- tripleo_ceph_client_exclude is defined
- tripleo_ceph_client_exclude | length > 0
# if include list is used, then exclude list does not matter, so then
# reset the effective clients fact; this task must be after exclude task
- name: set effective allow list with include
set_fact:
tripleo_ceph_client_effective_clients: "{{ client_group |
intersect(tripleo_ceph_client_include) |
list }}"
when:
- tripleo_ceph_client_include is defined
- tripleo_ceph_client_include | length > 0

View File

@ -57,6 +57,11 @@
- ceph_input_vars.stat.exists == False
- ceph_ansible_group_vars_all is defined
- name: Configure multiple external ceph clusters
include_tasks: multiple_external_ceph_clusters.yml
when:
- multiple is defined and multiple | length > 0
- include_tasks: create_keys.yml
name: Render keys provided for the Ceph cluster
vars:

View File

@ -0,0 +1,26 @@
---
# Copyright 2020 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Get values for multiple external ceph clusters
block:
- name: Prepare ceph config variables
set_fact:
external_cluster_mon_ips: "{{ multiple.external_cluster_mon_ips }}"
tripleo_ceph_client_fsid: "{{ multiple.fsid }}"
tripleo_ceph_client_cluster_network: ''
tripleo_ceph_client_mon_ips: []
tripleo_ceph_client_cluster: "{{ multiple.cluster }}"
tripleo_ceph_client_keys: "{{ multiple.get('keys',[]) }}"

View File

@ -17,11 +17,38 @@
- name: copy keys around
tags: 'ceph_client_rsync'
block:
- name: set effective ceph_clients without limit
set_fact:
tripleo_ceph_client_effective_clients: "{{ groups['ceph_client'] }}"
when:
- (ansible_limit is not defined) or (ansible_limit|length < 1)
- name: set effective ceph_clients with limit
include_tasks: effective_clients_limit.yml
vars:
client_group: "{{ groups['ceph_client'] }}"
when:
- (ansible_limit is defined) and (ansible_limit|length > 0)
# The ceph_external_multi_config usecase requires mon_client_hosts
# which is defined in roles/tripleo_ceph_work_dir/tasks/prepare.yml
- name: set distribution list from cross product of files and effective clients
set_fact:
tripleo_ceph_client_dist: "{{ lookup('fileglob',
tripleo_ceph_client_fetch_dir ~ '/*',
wantlist=True)
| product(mon_client_hosts|default([])
| union(tripleo_ceph_client_effective_clients | default([]))
| unique)
| list }}"
- name: Ensure /etc/ceph exists on all clients
become: true
file:
path: "{{ tripleo_ceph_client_config_home }}"
state: "directory"
delegate_to: "{{ item[1] }}"
loop: "{{ tripleo_ceph_client_dist }}"
- name: push files to the other nodes of cluster
become: true
@ -29,4 +56,4 @@
src: "{{ item[0] }}"
dest: "{{ tripleo_ceph_client_config_home }}/{{ item[0] | basename }}"
delegate_to: "{{ item[1] }}"
loop: "{{ lookup('fileglob', '{{ tripleo_ceph_client_fetch_dir }}/*', wantlist=True) |product(groups['ceph_client'])|list }}"
loop: "{{ tripleo_ceph_client_dist }}"