From 9f28e37aeb99388e9fd0f6f9722a6b3c810bb7c8 Mon Sep 17 00:00:00 2001 From: John Fulton Date: Wed, 13 Jan 2021 22:42:04 +0000 Subject: [PATCH] Introduce tripleo_cephadm role This role is provided as part of the implementation of the tripleo-ceph spec. It is an Ansible wrapper to call the cephadm Ceph tool and it contains the Ansible modules ceph_key and ceph_pool from ceph-ansible for managing cephx keys and ceph pools. Implements: blueprint tripleo-ceph Change-Id: I60d6857b888ef97242c4f4bbf20fbc62de5ef29f --- .pre-commit-config.yaml | 1 + doc/source/roles/role-tripleo_cephadm.rst | 117 +++ .../ansible_plugins/modules/ceph_key.py | 748 +++++++++++++++++ .../ansible_plugins/modules/ceph_pool.py | 766 ++++++++++++++++++ .../roles/tripleo_cephadm/defaults/main.yml | 36 + .../roles/tripleo_cephadm/files/.gitkeep | 0 .../tripleo_cephadm/files/ceph_spec.yaml | 18 + .../roles/tripleo_cephadm/meta/main.yml | 42 + .../molecule/default/Dockerfile | 37 + .../molecule/default/converge.yml | 50 ++ .../molecule/default/mock_ceph_keys.yml | 109 +++ .../molecule/default/mock_ceph_mon_dump.json | 92 +++ .../molecule/default/molecule.yml | 48 ++ .../molecule/default/prepare.yml | 40 + .../molecule/default/verify.yml | 90 ++ .../tripleo_cephadm/tasks/apply_spec.yaml | 55 ++ .../tripleo_cephadm/tasks/bootstrap.yaml | 83 ++ .../roles/tripleo_cephadm/tasks/ceph_cli.yaml | 27 + .../roles/tripleo_cephadm/tasks/export.yaml | 80 ++ .../roles/tripleo_cephadm/tasks/keys.yaml | 38 + .../roles/tripleo_cephadm/tasks/main.yml | 2 + .../roles/tripleo_cephadm/tasks/pools.yaml | 41 + .../roles/tripleo_cephadm/tasks/pre.yaml | 113 +++ .../tasks/wait_for_expected_num_mons.yaml | 44 + .../tripleo_cephadm/templates/ceph.conf.j2 | 13 + .../templates/ceph_client.yaml.j2 | 14 + zuul.d/molecule.yaml | 31 +- 27 files changed, 2724 insertions(+), 11 deletions(-) create mode 100644 doc/source/roles/role-tripleo_cephadm.rst create mode 100644 tripleo_ansible/ansible_plugins/modules/ceph_key.py create mode 100644 tripleo_ansible/ansible_plugins/modules/ceph_pool.py create mode 100644 tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/files/.gitkeep create mode 100644 tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/meta/main.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/molecule/default/Dockerfile create mode 100644 tripleo_ansible/roles/tripleo_cephadm/molecule/default/converge.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock_ceph_keys.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock_ceph_mon_dump.json create mode 100644 tripleo_ansible/roles/tripleo_cephadm/molecule/default/molecule.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/molecule/default/prepare.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/molecule/default/verify.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/bootstrap.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/export.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/keys.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/main.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/pools.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/pre.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/wait_for_expected_num_mons.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/templates/ceph.conf.j2 create mode 100644 tripleo_ansible/roles/tripleo_cephadm/templates/ceph_client.yaml.j2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3662e41be..1cc62df1b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,7 @@ repos: - id: debug-statements - id: check-yaml files: .*\.(yaml|yml)$ + args: [--allow-multiple-documents] - repo: https://gitlab.com/pycqa/flake8 rev: 3.8.3 hooks: diff --git a/doc/source/roles/role-tripleo_cephadm.rst b/doc/source/roles/role-tripleo_cephadm.rst new file mode 100644 index 000000000..1be09d97f --- /dev/null +++ b/doc/source/roles/role-tripleo_cephadm.rst @@ -0,0 +1,117 @@ +====================== +Role - tripleo_cephadm +====================== + +.. ansibleautoplugin:: + :role: tripleo_ansible/roles/tripleo_cephadm + +About +~~~~~ + +An Ansible role for TripleO integration with Ceph clusters deployed with +`cephadm`_ and managed with Ceph `orchestrator`_. + +This role is provided as part of the implementation of the `tripleo_ceph_spec`_. +It is an Ansible wrapper to call the Ceph tools `cephadm`_ and `orchestrator`_ +and it contains the Ansible module `ceph_key`_ from `ceph-ansible`_. + +Assumptions +~~~~~~~~~~~ + +- This role assumes it has an inventory with a single host, known as the + `bootstrap_host`. An inventory genereated by `tripleo-ansible-inventory` + will have a `mons` group so the first node in this group is a good + candidate for this host. + +- The `cephadm`_ binary must be installed on the `bootstrap_host`. + +- Though there only needs to be one Ceph node in the inventory `cephadm`_ + will configure the other servers with SSH. Thus, the following playbook + should be run before one which uses this role to configure the `ceph-admin` + user on the overcloud with the SSH keys that `cephadm`_ requires. + + .. code-block:: bash + + ansible-playbook -i $INV \ + tripleo-ansible/tripleo_ansible/playbooks/cli-enable-ssh-admin.yaml \ + -e @ceph-admin.yml + + Where `ceph-admin.yml` contains something like the following: + + .. code-block:: YAML + + --- + tripleo_admin_user: ceph-admin + ssh_servers: "{{ groups['mons'] }}" + distribute_private_key: true + + The `ssh_servers` variable should be expanded to contain another other nodes + hosting Ceph, e.g. `osds`. + +- A `cephadm-spec`_ file should be provided which references the Ceph services + to be run on the other `ssh_hosts`. The path to this file can be set with + the `ceph_spec` variable. + +Usage +~~~~~ + +Here is an example of a playbook which bootstraps the first Ceph monitor +and then applies a spec file to add other hosts. It then creates RBD pools +for Nova, Cinder, and Glance and a cephx keyring called `openstack` to access +those pools. It then creates a file which can be passed as input to the role +`tripleo_ceph_client` so that an overcloud can be configured to use the deployed +Ceph cluster. + +.. code-block:: YAML + + - name: Deploy Ceph with cephadm + hosts: mons[0] + vars: + bootstrap_host: "{{ groups['mons'][0] }}" + tripleo_cephadm_spec_on_bootstrap: false + pools: + - vms + - volumes + - images + tasks: + - name: Satisfy Ceph prerequisites + import_role: + role: tripleo_cephadm + tasks_from: pre + + - name: Bootstrap Ceph + import_role: + role: tripleo_cephadm + tasks_from: bootstrap + + - name: Apply Ceph spec + import_role: + role: tripleo_cephadm + tasks_from: apply_spec + when: not tripleo_cephadm_spec_on_bootstrap + + - name: Create Pools + import_role: + role: tripleo_cephadm + tasks_from: pools + + - name: Create Keys + import_role: + role: tripleo_cephadm + tasks_from: keys + + - name: Export configuration for tripleo_ceph_client + import_role: + role: tripleo_cephadm + tasks_from: export + vars: + cephx_keys: + - client.openstack + + +.. _tripleo_ceph_spec: https://specs.openstack.org/openstack/tripleo-specs/specs/wallaby/tripleo-ceph.html +.. _cephadm: https://docs.ceph.com/en/latest/cephadm/ +.. _orchestrator: https://docs.ceph.com/en/latest/mgr/orchestrator/ +.. _ceph_key: https://github.com/ceph/ceph-ansible/blob/master/library/ceph_key.py +.. _ceph-ansible: https://github.com/ceph/ceph-ansible/ +.. _cephadm-spec: https://tracker.ceph.com/issues/44205 diff --git a/tripleo_ansible/ansible_plugins/modules/ceph_key.py b/tripleo_ansible/ansible_plugins/modules/ceph_key.py new file mode 100644 index 000000000..0457fc28b --- /dev/null +++ b/tripleo_ansible/ansible_plugins/modules/ceph_key.py @@ -0,0 +1,748 @@ +#!/usr/bin/python3 + +# Copyright 2018, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Included from: https://github.com/ceph/ceph-ansible/blob/master/library/ceph_key.py + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from ansible.module_utils.basic import AnsibleModule +import datetime +import json +import yaml +import os +import struct +import time +import base64 +import socket + + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = """ +--- +module: ceph_key +author: Sebastien Han +short_description: Manage Cephx key(s) +version_added: "2.6" +notes: [] +description: + - Manage CephX creation, deletion and updates. + It can also list and get information about keyring(s). +requirements: + - None +options: + cluster: + description: + - The ceph cluster name. + required: false + type: str + default: ceph + name: + description: + - name of the CephX key + type: str + required: true + user: + description: + - entity used to perform operation. + It corresponds to the -n option (--name) + type: str + required: false + default: client.admin + user_key: + description: + - the path to the keyring corresponding to the + user being used. It corresponds to the -k + option (--keyring) + type: str + state: + description: + - If 'present' is used, the module creates a keyring + with the associated capabilities. + If 'present' is used and a secret is provided the module + will always add the key. Which means it will update + the keyring if the secret changes, the same goes for + the capabilities. + If 'absent' is used, the module will simply delete the keyring. + If 'list' is used, the module will list all the keys and will + return a json output. + If 'info' is used, the module will return in a json format the + description of a given keyring. + If 'generate_secret' is used, the module will simply output a cephx keyring. + required: false + type: str + choices: ['present', 'update', 'absent', 'list', 'info', 'fetch_initial_keys', 'generate_secret'] + default: 'present' + caps: + description: + - CephX key capabilities + type: dict + required: false + secret: + description: + - keyring's secret value + required: false + type: str + import_key: + description: + - Wether or not to import the created keyring into Ceph. + This can be useful for someone that only wants to generate keyrings + but not add them into Ceph. + required: false + type: bool + default: true + dest: + description: + - Destination to write the keyring, can a file or a directory + required: false + type: str + default: '/etc/ceph/' + fetch_initial_keys: + description: + - Fetch client.admin and bootstrap key. + This is only needed for Nautilus and above. + Writes down to the filesystem the initial keys generated by the monitor. + This command can ONLY run from a monitor node. + required: false + type: str + default: 'false' + output_format: + description: + - The key output format when retrieving the information of an + entity. + required: false + type: str + default: 'json' +""" + +EXAMPLES = ''' + +keys_to_create: + - { name: client.key, key: "AQAin8tUUK84ExAA/QgBtI7gEMWdmnvKBzlXdQ==", \ + caps: { mon: "allow rwx", mds: "allow *" } , mode: "0600" } + - { name: client.cle, caps: { mon: "allow r", osd: "allow *" } , mode: "0600" } + +caps: + mon: "allow rwx" + mds: "allow *" + +- name: create ceph admin key + ceph_key: + name: client.admin + state: present + secret: AQAin8tU2DsKFBAAFIAzVTzkL3+gtAjjpQiomw== + caps: + mon: allow * + osd: allow * + mgr: allow * + mds: allow + mode: 0400 + import_key: false + +- name: create monitor initial keyring + ceph_key: + name: mon. + state: present + secret: AQAin8tUMICVFBAALRHNrV0Z4MXupRw4v9JQ6Q== + caps: + mon: allow * + dest: "/var/lib/ceph/tmp/" + import_key: false + +- name: create cephx key + ceph_key: + name: "{{ keys_to_create }}" + user: client.bootstrap-rgw + user_key: /var/lib/ceph/bootstrap-rgw/ceph.keyring + state: present + caps: "{{ caps }}" + +- name: create cephx key but don't import it in Ceph + ceph_key: + name: "{{ keys_to_create }}" + state: present + caps: "{{ caps }}" + import_key: false + +- name: delete cephx key + ceph_key: + name: "my_key" + state: absent + +- name: info cephx key + ceph_key: + name: "my_key"" + state: info + +- name: info cephx admin key (plain) + ceph_key: + name: client.admin + output_format: plain + state: info + register: client_admin_key + +- name: list cephx keys + ceph_key: + state: list + +- name: fetch cephx keys + ceph_key: + state: fetch_initial_keys +''' + +RETURN = '''# ''' + + +CEPH_INITIAL_KEYS = ['client.admin', + 'client.bootstrap-mds', 'client.bootstrap-mgr', + 'client.bootstrap-osd', 'client.bootstrap-rbd', + 'client.bootstrap-rbd-mirror', 'client.bootstrap-rgw'] + + +def fatal(message, module): + ''' + Report a fatal error and exit + ''' + + if module: + module.fail_json(msg=message, rc=1) + else: + raise(Exception(message)) + + +def generate_secret(): + ''' + Generate a CephX secret + ''' + + key = os.urandom(16) + header = struct.pack(' +short_description: Manage Ceph Pools +version_added: "2.8" +description: + - Manage Ceph pool(s) creation, deletion and updates. +options: + cluster: + description: + - The ceph cluster name. + required: false + default: ceph + type: str + name: + description: + - name of the Ceph pool + required: true + type: str + state: + description: + If 'present' is used, the module creates a pool if it doesn't exist + or update it if it already exists. + If 'absent' is used, the module will simply delete the pool. + If 'list' is used, the module will return all details about the + existing pools. (json formatted). + required: false + type: str + choices: ['present', 'absent', 'list'] + default: present + size: + description: + - set the replica size of the pool. + required: false + type: str + min_size: + description: + - set the min_size parameter of the pool. + required: false + type: str + pg_num: + description: + - set the pg_num of the pool. + required: false + type: str + pgp_num: + description: + - set the pgp_num of the pool. + required: false + type: str + pg_autoscale_mode: + description: + - set the pg autoscaler on the pool. + required: false + default: 'on' + type: str + target_size_ratio: + description: + - set the target_size_ratio on the pool + required: false + type: str + pool_type: + description: + - set the pool type, either 'replicated' or 'erasure' + required: false + default: 'replicated' + type: str + erasure_profile: + description: + - When pool_type = 'erasure', set the erasure profile of the pool + required: false + default: 'default' + type: str + rule_name: + description: + - Set the crush rule name assigned to the pool + required: false + default: 'replicated_rule' + type: str + expected_num_objects: + description: + - Set the expected_num_objects parameter of the pool. + required: false + default: '0' + application: + description: + - Set the pool application on the pool. + required: false + type: str +""" + +EXAMPLES = ''' + +pools: + - { name: foo, size: 3, application: rbd, pool_type: 'replicated', + pg_autoscale_mode: 'on' } + +- hosts: all + become: true + tasks: + - name: create a pool + ceph_pool: + name: "{{ item.name }}" + state: present + size: "{{ item.size }}" + application: "{{ item.application }}" + pool_type: "{{ item.pool_type }}" + pg_autoscale_mode: "{{ item.pg_autoscale_mode }}" + with_items: "{{ pools }}" +''' + +RETURN = '''# ''' + +# Start TripleO change +# Tripleo only needs ca_common module_utils for this module. +# Rather than add to tripleo-ansible's module_utils, insert 6 functions here +# https://github.com/ceph/ceph-ansible/blob/master/module_utils/ca_common.py + + +def generate_ceph_cmd(sub_cmd, args, user_key=None, + cluster='ceph', user='client.admin', + container_image=None, interactive=False): + ''' + Generate 'ceph' command line to execute + ''' + + if not user_key: + user_key = '/etc/ceph/{}.{}.keyring'.format(cluster, user) + + cmd = pre_generate_ceph_cmd(container_image=container_image, interactive=interactive) + + base_cmd = [ + '-n', + user, + '-k', + user_key, + '--cluster', + cluster + ] + base_cmd.extend(sub_cmd) + cmd.extend(base_cmd + args) + + return cmd + + +def container_exec(binary, container_image, interactive=False): + ''' + Build the docker CLI to run a command inside a container + ''' + + container_binary = os.getenv('CEPH_CONTAINER_BINARY') + command_exec = [container_binary, 'run'] + + if interactive: + command_exec.extend(['--interactive']) + + command_exec.extend(['--rm', + '--net=host', + '-v', '/etc/ceph:/etc/ceph:z', + '-v', '/var/lib/ceph/:/var/lib/ceph/:z', + '-v', '/var/log/ceph/:/var/log/ceph/:z', + '--entrypoint={}'.format(binary), container_image]) + return command_exec + + +def is_containerized(): + ''' + Check if we are running on a containerized cluster + ''' + + if 'CEPH_CONTAINER_IMAGE' in os.environ: + container_image = os.getenv('CEPH_CONTAINER_IMAGE') + else: + container_image = None + + return container_image + + +def pre_generate_ceph_cmd(container_image=None, interactive=False): + ''' + Generate ceph prefix comaand + ''' + if container_image: + cmd = container_exec('ceph', container_image, interactive=interactive) + else: + cmd = ['ceph'] + + return cmd + + +def exec_command(module, cmd, stdin=None): + ''' + Execute command(s) + ''' + + binary_data = False + if stdin: + binary_data = True + rc, out, err = module.run_command(cmd, data=stdin, binary_data=binary_data) + + return rc, cmd, out, err + + +def exit_module(module, out, rc, cmd, err, startd, changed=False): + endd = datetime.datetime.now() + delta = endd - startd + + result = dict( + cmd=cmd, + start=str(startd), + end=str(endd), + delta=str(delta), + rc=rc, + stdout=out.rstrip("\r\n"), + stderr=err.rstrip("\r\n"), + changed=changed, + ) + module.exit_json(**result) +# End TripleO change + + +def check_pool_exist(cluster, + name, + user, + user_key, + output_format='json', + container_image=None): + ''' + Check if a given pool exists + ''' + + args = ['stats', name, '-f', output_format] + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + return cmd + + +def generate_get_config_cmd(param, + cluster, + user, + user_key, + container_image=None): + _cmd = pre_generate_ceph_cmd(container_image=container_image) + args = [ + '-n', + user, + '-k', + user_key, + '--cluster', + cluster, + 'config', + 'get', + 'mon.*', + param + ] + cmd = _cmd + args + return cmd + + +def get_application_pool(cluster, + name, + user, + user_key, + output_format='json', + container_image=None): + ''' + Get application type enabled on a given pool + ''' + + args = ['application', 'get', name, '-f', output_format] + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + return cmd + + +def enable_application_pool(cluster, + name, + application, + user, + user_key, + container_image=None): + ''' + Enable application on a given pool + ''' + + args = ['application', 'enable', name, application] + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + return cmd + + +def disable_application_pool(cluster, + name, + application, + user, + user_key, + container_image=None): + ''' + Disable application on a given pool + ''' + + args = ['application', 'disable', name, + application, '--yes-i-really-mean-it'] + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + return cmd + + +def get_pool_details(module, + cluster, + name, + user, + user_key, + output_format='json', + container_image=None): + ''' + Get details about a given pool + ''' + + args = ['ls', 'detail', '-f', output_format] + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + rc, cmd, out, err = exec_command(module, cmd) + + if rc == 0: + out = [p for p in json.loads(out.strip()) if p['pool_name'] == name][0] + + _rc, _cmd, application_pool, _err = exec_command(module, + get_application_pool(cluster, # noqa: E501 + name, # noqa: E501 + user, # noqa: E501 + user_key, # noqa: E501 + container_image=container_image)) # noqa: E501 + + # This is a trick because "target_size_ratio" isn't present at the same level in the dict + # ie: + # { + # 'pg_num': 8, + # 'pgp_num': 8, + # 'pg_autoscale_mode': 'on', + # 'options': { + # 'target_size_ratio': 0.1 + # } + # } + # If 'target_size_ratio' is present in 'options', we set it, this way we end up + # with a dict containing all needed keys at the same level. + if 'target_size_ratio' in out['options'].keys(): + out['target_size_ratio'] = out['options']['target_size_ratio'] + else: + out['target_size_ratio'] = None + + application = list(json.loads(application_pool.strip()).keys()) + + if len(application) == 0: + out['application'] = '' + else: + out['application'] = application[0] + + return rc, cmd, out, err + + +def compare_pool_config(user_pool_config, running_pool_details): + ''' + Compare user input config pool details with current running pool details + ''' + + delta = {} + filter_keys = ['pg_num', 'pg_placement_num', 'size', + 'pg_autoscale_mode', 'target_size_ratio'] + for key in filter_keys: + if (str(running_pool_details[key]) != user_pool_config[key]['value'] and user_pool_config[key]['value']): + delta[key] = user_pool_config[key] + + if (running_pool_details['application'] != user_pool_config['application']['value'] and user_pool_config['application']['value']): + delta['application'] = {} + delta['application']['new_application'] = user_pool_config['application']['value'] # noqa: E501 + # to be improved (for update_pools()...) + delta['application']['value'] = delta['application']['new_application'] + delta['application']['old_application'] = running_pool_details['application'] # noqa: E501 + + return delta + + +def list_pools(cluster, + user, + user_key, + details, + output_format='json', + container_image=None): + ''' + List existing pools + ''' + + args = ['ls'] + + if details: + args.append('detail') + + args.extend(['-f', output_format]) + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + return cmd + + +def create_pool(cluster, + name, + user, + user_key, + user_pool_config, + container_image=None): + ''' + Create a new pool + ''' + + args = ['create', user_pool_config['pool_name']['value'], + user_pool_config['type']['value']] + + if user_pool_config['pg_autoscale_mode']['value'] != 'on': + args.extend(['--pg_num', + user_pool_config['pg_num']['value'], + '--pgp_num', + user_pool_config['pgp_num']['value']]) + elif user_pool_config['target_size_ratio']['value']: + args.extend(['--target_size_ratio', + user_pool_config['target_size_ratio']['value']]) + + if user_pool_config['type']['value'] == 'replicated': + args.extend([user_pool_config['crush_rule']['value'], + '--expected_num_objects', + user_pool_config['expected_num_objects']['value'], + '--autoscale-mode', + user_pool_config['pg_autoscale_mode']['value']]) + + if (user_pool_config['size']['value'] and user_pool_config['type']['value'] == "replicated"): + args.extend(['--size', user_pool_config['size']['value']]) + + elif user_pool_config['type']['value'] == 'erasure': + args.extend([user_pool_config['erasure_profile']['value']]) + + if user_pool_config['crush_rule']['value']: + args.extend([user_pool_config['crush_rule']['value']]) + + args.extend(['--expected_num_objects', + user_pool_config['expected_num_objects']['value'], + '--autoscale-mode', + user_pool_config['pg_autoscale_mode']['value']]) + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + return cmd + + +def remove_pool(cluster, name, user, user_key, container_image=None): + ''' + Remove a pool + ''' + + args = ['rm', name, name, '--yes-i-really-really-mean-it'] + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + return cmd + + +def update_pool(module, cluster, name, + user, user_key, delta, container_image=None): + ''' + Update an existing pool + ''' + + report = "" + + for key in delta.keys(): + if key != 'application': + args = ['set', + name, + delta[key]['cli_set_opt'], + delta[key]['value']] + + cmd = generate_ceph_cmd(sub_cmd=['osd', 'pool'], + args=args, + cluster=cluster, + user=user, + user_key=user_key, + container_image=container_image) + + rc, cmd, out, err = exec_command(module, cmd) + if rc != 0: + return rc, cmd, out, err + + else: + rc, cmd, out, err = exec_command(module, disable_application_pool(cluster, name, delta['application']['old_application'], user, user_key, container_image=container_image)) # noqa: E501 + if rc != 0: + return rc, cmd, out, err + + rc, cmd, out, err = exec_command(module, enable_application_pool(cluster, name, delta['application']['new_application'], user, user_key, container_image=container_image)) # noqa: E501 + if rc != 0: + return rc, cmd, out, err + + report = report + "\n" + "{} has been updated: {} is now {}".format(name, key, delta[key]['value']) # noqa: E501 + + out = report + return rc, cmd, out, err + + +def run_module(): + module = AnsibleModule( + argument_spec=yaml.safe_load(DOCUMENTATION)['options'], + supports_check_mode=True, + ) + + # Gather module parameters in variables + cluster = module.params.get('cluster') + name = module.params.get('name') + state = module.params.get('state') + details = module.params.get('details') + size = module.params.get('size') + min_size = module.params.get('min_size') + pg_num = module.params.get('pg_num') + pgp_num = module.params.get('pgp_num') + pg_autoscale_mode = module.params.get('pg_autoscale_mode') + target_size_ratio = module.params.get('target_size_ratio') + application = module.params.get('application') + + if (module.params.get('pg_autoscale_mode').lower() in + ['true', 'on', 'yes']): + pg_autoscale_mode = 'on' + elif (module.params.get('pg_autoscale_mode').lower() in + ['false', 'off', 'no']): + pg_autoscale_mode = 'off' + else: + pg_autoscale_mode = 'warn' + + if module.params.get('pool_type') == '1': + pool_type = 'replicated' + elif module.params.get('pool_type') == '3': + pool_type = 'erasure' + else: + pool_type = module.params.get('pool_type') + + if not module.params.get('rule_name'): + rule_name = 'replicated_rule' if pool_type == 'replicated' else None + else: + rule_name = module.params.get('rule_name') + + erasure_profile = module.params.get('erasure_profile') + expected_num_objects = module.params.get('expected_num_objects') + user_pool_config = { + 'pool_name': {'value': name}, + 'pg_num': {'value': pg_num, 'cli_set_opt': 'pg_num'}, + 'pgp_num': {'value': pgp_num, 'cli_set_opt': 'pgp_num'}, + 'pg_autoscale_mode': {'value': pg_autoscale_mode, + 'cli_set_opt': 'pg_autoscale_mode'}, + 'target_size_ratio': {'value': target_size_ratio, + 'cli_set_opt': 'target_size_ratio'}, + 'application': {'value': application}, + 'type': {'value': pool_type}, + 'erasure_profile': {'value': erasure_profile}, + 'crush_rule': {'value': rule_name, 'cli_set_opt': 'crush_rule'}, + 'expected_num_objects': {'value': expected_num_objects}, + 'size': {'value': size, 'cli_set_opt': 'size'}, + 'min_size': {'value': min_size} + } + + if module.check_mode: + module.exit_json( + changed=False, + stdout='', + stderr='', + rc=0, + start='', + end='', + delta='', + ) + + startd = datetime.datetime.now() + changed = False + + # will return either the image name or None + container_image = is_containerized() + + user = "client.admin" + keyring_filename = cluster + '.' + user + '.keyring' + user_key = os.path.join("/etc/ceph/", keyring_filename) + + if state == "present": + rc, cmd, out, err = exec_command(module, + check_pool_exist(cluster, + name, + user, + user_key, + container_image=container_image)) # noqa: E501 + if rc == 0: + running_pool_details = get_pool_details(module, + cluster, + name, + user, + user_key, + container_image=container_image) # noqa: E501 + user_pool_config['pg_placement_num'] = {'value': str(running_pool_details[2]['pg_placement_num']), 'cli_set_opt': 'pgp_num'} # noqa: E501 + delta = compare_pool_config(user_pool_config, + running_pool_details[2]) + if len(delta) > 0: + keys = list(delta.keys()) + details = running_pool_details[2] + if details['erasure_code_profile'] and 'size' in keys: + del delta['size'] + if details['pg_autoscale_mode'] == 'on': + delta.pop('pg_num', None) + delta.pop('pgp_num', None) + + if len(delta) == 0: + out = "Skipping pool {}.\nUpdating either 'size' on an erasure-coded pool " \ + "or 'pg_num'/'pgp_num' on a pg autoscaled pool is incompatible".format(name) + else: + rc, cmd, out, err = update_pool(module, + cluster, + name, + user, + user_key, + delta, + container_image=container_image) # noqa: E501 + if rc == 0: + changed = True + else: + out = "Pool {} already exists and there is nothing to update.".format(name) # noqa: E501 + else: + rc, cmd, out, err = exec_command(module, + create_pool(cluster, + name, + user, + user_key, + user_pool_config=user_pool_config, # noqa: E501 + container_image=container_image)) # noqa: E501 + if user_pool_config['application']['value']: + rc, _, _, _ = exec_command(module, + enable_application_pool(cluster, + name, + user_pool_config['application']['value'], # noqa: E501 + user, + user_key, + container_image=container_image)) # noqa: E501 + if user_pool_config['min_size']['value']: + # not implemented yet + pass + changed = True + + elif state == "list": + rc, cmd, out, err = exec_command(module, + list_pools(cluster, + name, user, + user_key, + details, + container_image=container_image)) # noqa: E501 + if rc != 0: + out = "Couldn't list pool(s) present on the cluster" + + elif state == "absent": + rc, cmd, out, err = exec_command(module, + check_pool_exist(cluster, + name, user, + user_key, + container_image=container_image)) # noqa: E501 + if rc == 0: + rc, cmd, out, err = exec_command(module, + remove_pool(cluster, + name, + user, + user_key, + container_image=container_image)) # noqa: E501 + changed = True + else: + rc = 0 + out = "Skipped, since pool {} doesn't exist".format(name) + + exit_module(module=module, out=out, rc=rc, cmd=cmd, err=err, startd=startd, + changed=changed) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml new file mode 100644 index 000000000..2dd69cdb9 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml @@ -0,0 +1,36 @@ +--- +# defaults file for tripleo_cephadm +tripleo_cephadm_spec_on_bootstrap: false # not recommended due to https://tracker.ceph.com/issues/49277 +tripleo_cephadm_ssh_user: ceph-admin +tripleo_cephadm_bin: /usr/sbin/cephadm +tripleo_cephadm_cluster: ceph +tripleo_cephadm_config_home: /etc/ceph +tripleo_cephadm_verbose: true +tripleo_cephadm_container_ns: "docker.io/ceph" +tripleo_cephadm_container_image: "ceph" +tripleo_cephadm_container_tag: "v15" +tripleo_cephadm_container_cli: "podman" +tripleo_cephadm_container_options: "--net=host --ipc=host" +tripleo_cephadm_admin_keyring: "{{ tripleo_cephadm_config_home }}/{{ tripleo_cephadm_cluster }}.client.admin.keyring" +tripleo_cephadm_conf: "{{ tripleo_cephadm_config_home }}/{{ tripleo_cephadm_cluster }}.conf" +tripleo_cephadm_bootstrap_conf: "/home/{{ tripleo_cephadm_ssh_user }}/bootstrap_{{ tripleo_cephadm_cluster }}.conf" +tripleo_cephadm_spec: "/home/{{ tripleo_cephadm_ssh_user }}/specs/ceph_spec.yaml" +tripleo_cephadm_container_spec: /home/ceph_spec.yaml +tripleo_cephadm_spec_ansible_host: "{{ role_path }}/files/ceph_spec.yaml" +tripleo_cephadm_bootstrap_files: + - "/home/{{ tripleo_cephadm_ssh_user }}/.ssh/id_rsa" + - "/home/{{ tripleo_cephadm_ssh_user }}/.ssh/id_rsa.pub" +tripleo_cephadm_uid: "167" +tripleo_cephadm_mode: "0755" +tripleo_cephadm_keyring_permissions: "0644" +tripleo_ceph_client_vars: "/home/stack/ceph_client.yaml" +tripleo_cephadm_dashboard_enabled: false +tripleo_cephadm_wait_for_mons: true +tripleo_cephadm_wait_for_mons_retries: 10 +tripleo_cephadm_wait_for_mons_delay: 20 +tripleo_cephadm_wait_for_mons_ignore_errors: false +tripleo_cephadm_predeployed: true +tripleo_cephadm_conf_overrides: {} +tripleo_cephadm_fsid_list: [] +# todo(fultonj) add is_hci boolean for target memory +# https://lists.ceph.io/hyperkitty/list/dev@ceph.io/thread/Z77XO23JPXDNHKM7IG6UN4URYKA6L7VH/ diff --git a/tripleo_ansible/roles/tripleo_cephadm/files/.gitkeep b/tripleo_ansible/roles/tripleo_cephadm/files/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml b/tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml new file mode 100644 index 000000000..bcd6585fa --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml @@ -0,0 +1,18 @@ +--- +service_type: host +addr: standalone.localdomain +hostname: standalone.localdomain +--- +service_type: mon +placement: + hosts: + - standalone.localdomain +--- +service_type: osd +service_id: standalone_drive_group +placement: + hosts: + - standalone.localdomain +data_devices: + paths: + - /dev/ceph_vg/ceph_lv_data diff --git a/tripleo_ansible/roles/tripleo_cephadm/meta/main.yml b/tripleo_ansible/roles/tripleo_cephadm/meta/main.yml new file mode 100644 index 000000000..6a39bdf26 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/meta/main.yml @@ -0,0 +1,42 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +galaxy_info: + author: OpenStack + description: TripleO OpenStack Role -- tripleo_cephadm + company: Red Hat + license: Apache-2.0 + min_ansible_version: 2.7 + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + platforms: + - name: CentOS + versions: + - 7 + - 8 + + galaxy_tags: + - tripleo + + +# List your role dependencies here, one per line. Be sure to remove the '[]' above, +# if you add dependencies to this list. +dependencies: [] diff --git a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/Dockerfile b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/Dockerfile new file mode 100644 index 000000000..90646fff5 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/Dockerfile @@ -0,0 +1,37 @@ +# Molecule managed +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi + +{% for pkg in item.easy_install | default([]) %} +# install pip for centos where there is no python-pip rpm in default repos +RUN easy_install {{ pkg }} +{% endfor %} + + +CMD ["sh", "-c", "while true; do sleep 10000; done"] diff --git a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/converge.yml b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/converge.yml new file mode 100644 index 000000000..0126ef082 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/converge.yml @@ -0,0 +1,50 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Converge + hosts: all + vars: + tripleo_cephadm_wait_for_mons: false + tripleo_ceph_client_vars: ceph_client.yaml + tasks: + - name: Satisfy Ceph prerequisites + import_role: + name: tripleo_cephadm + tasks_from: pre + + - name: Bootstrap Ceph + import_role: + name: tripleo_cephadm + tasks_from: bootstrap + + - name: Mock ceph_mon_dump command + shell: "cat mock_ceph_mon_dump.json" + register: ceph_mon_mock_dump + delegate_to: localhost + + - name: Mock ceph_keys_module_output + include_vars: mock_ceph_keys.yml + + - name: Export configuration for tripleo_ceph_client + import_role: + name: tripleo_cephadm + tasks_from: export + vars: + ceph_mon_dump: "{{ ceph_mon_mock_dump }}" + tripleo_cephadm_client_keys: "{{ mock_ceph_keys }}" + + - name: Run verify tasks + include_tasks: verify.yml diff --git a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock_ceph_keys.yml b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock_ceph_keys.yml new file mode 100644 index 000000000..c0313af82 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock_ceph_keys.yml @@ -0,0 +1,109 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +mock_ceph_keys: + results: + - ansible_facts: + discovered_interpreter_python: /usr/libexec/platform-python + ansible_loop_var: item + changed: false + cmd: + - podman + - run + - --rm + - --net=host + - -v + - /etc/ceph:/etc/ceph:z + - -v + - /var/lib/ceph/:/var/lib/ceph/:z + - -v + - /var/log/ceph/:/var/log/ceph/:z + - --entrypoint=ceph + - undercloud.ctlplane.mydomain.tld:8787/ceph-ci/daemon:v5.0.7-stable-5.0-octopus-centos-8-x86_64 + - -n + - client.admin + - -k + - /etc/ceph/ceph.client.admin.keyring + - --cluster + - ceph + - auth + - get + - client.openstack + - -f + - json + delta: '0:00:01.501594' + end: '2021-01-29 19:48:16.372821' + failed: false + invocation: + module_args: + attributes: null + backup: null + caps: null + cluster: ceph + content: null + delimiter: null + dest: /etc/ceph/ + directory_mode: null + follow: false + force: null + group: null + import_key: true + mode: null + name: client.openstack + output_format: json + owner: null + regexp: null + remote_src: null + secret: null + selevel: null + serole: null + setype: null + seuser: null + src: null + state: info + unsafe_writes: null + user: client.admin + user_key: null + item: + caps: + mgr: allow * + mon: profile rbd + osd: profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images + key: AQATZBBgAAAAABAAUl/GZvcldk6G74AoZ2v2rg== + mode: '0600' + name: client.openstack + rc: 0 + start: '2021-01-29 19:48:14.871227' + stderr: exported keyring for client.openstack + stderr_lines: + - exported keyring for client.openstack + stdout: >- + [{"entity":"client.openstack", + "key":"AQATZBBgAAAAABAAUl/GZvcldk6G74AoZ2v2rg==", + "caps":{ + "mgr":"allow *", + "mon":"profile rbd", + "osd":"profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images" + }}] + stdout_lines: >- + - '' + - '[{"entity":"client.openstack", + "key":"AQATZBBgAAAAABAAUl/GZvcldk6G74AoZ2v2rg==", + "caps":{ + "mgr":"allow *", + "mon":"profile rbd", + "osd":"profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images" + }}]' diff --git a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock_ceph_mon_dump.json b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock_ceph_mon_dump.json new file mode 100644 index 000000000..17444ee82 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock_ceph_mon_dump.json @@ -0,0 +1,92 @@ +{ + "epoch": 3, + "fsid": "ca9bf37b-ed0f-4e5a-bb21-e5b5f9b75135", + "modified": "2021-01-26T19:23:44.536193Z", + "created": "2021-01-26T19:19:33.377161Z", + "min_mon_release": 15, + "min_mon_release_name": "octopus", + "features": { + "persistent": [ + "kraken", + "luminous", + "mimic", + "osdmap-prune", + "nautilus", + "octopus" + ], + "optional": [] + }, + "mons": [ + { + "rank": 0, + "name": "oc0-controller-0", + "public_addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "172.16.11.241:3300", + "nonce": 0 + }, + { + "type": "v1", + "addr": "172.16.11.241:6789", + "nonce": 0 + } + ] + }, + "addr": "172.16.11.241:6789/0", + "public_addr": "172.16.11.241:6789/0", + "priority": 0, + "weight": 0 + }, + { + "rank": 1, + "name": "oc0-controller-1", + "public_addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "172.16.11.176:3300", + "nonce": 0 + }, + { + "type": "v1", + "addr": "172.16.11.176:6789", + "nonce": 0 + } + ] + }, + "addr": "172.16.11.176:6789/0", + "public_addr": "172.16.11.176:6789/0", + "priority": 0, + "weight": 0 + }, + { + "rank": 2, + "name": "oc0-controller-2", + "public_addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "172.16.11.82:3300", + "nonce": 0 + }, + { + "type": "v1", + "addr": "172.16.11.82:6789", + "nonce": 0 + } + ] + }, + "addr": "172.16.11.82:6789/0", + "public_addr": "172.16.11.82:6789/0", + "priority": 0, + "weight": 0 + } + ], + "quorum": [ + 0, + 1, + 2 + ] +} diff --git a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/molecule.yml b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/molecule.yml new file mode 100644 index 000000000..f448198fe --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/molecule.yml @@ -0,0 +1,48 @@ +--- +driver: + name: podman + +log: true + +platforms: + - name: ubi8 + hostname: ubi8 + image: ubi8/ubi-init + registry: + url: registry.access.redhat.com + dockerfile: Dockerfile + pkg_extras: python*setuptools + volumes: + - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro + - /etc/pki/rpm-gpg:/etc/pki/rpm-gpg + privileged: true + environment: &env + http_proxy: "{{ lookup('env', 'http_proxy') }}" + https_proxy: "{{ lookup('env', 'https_proxy') }}" + ulimits: &ulimit + - host + +provisioner: + inventory: + hosts: + all: + hosts: + ubi8: + ansible_python_interpreter: /usr/bin/python3 + name: ansible + log: true + env: + ANSIBLE_STDOUT_CALLBACK: yaml + +scenario: + test_sequence: + - destroy + - create + - prepare + - converge + - check + - verify + - destroy + +verifier: + name: testinfra diff --git a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/prepare.yml b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/prepare.yml new file mode 100644 index 000000000..d3b6519c0 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/prepare.yml @@ -0,0 +1,40 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +- name: Prepare + hosts: all + roles: + - role: test_deps + tasks: + - name: Install additional dependencies + package: + name: openssh + state: present + - name: Enable ceph repsitory + command: tripleo-repos -b master current-tripleo ceph + become: true + - name: Ensure ceph-admin group exists + group: + name: ceph-admin + state: present + - name: Ensure ceph-admin user exists + user: + name: ceph-admin + comment: ceph-admin + group: ceph-admin + groups: wheel + generate_ssh_key: true diff --git a/tripleo_ansible/roles/tripleo_cephadm/molecule/default/verify.yml b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/verify.yml new file mode 100644 index 000000000..6a430f223 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/molecule/default/verify.yml @@ -0,0 +1,90 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Fail if the FSID is not defined or not valid + fail: + msg: "The tripleo_cephadm_fsid {{ tripleo_ceph_client_fsid|default('') }} variable is either undefined or not valid" + when: + - tripleo_cephadm_fsid is not defined + - tripleo_cephadm_fsid | regex_search(regex) + vars: + - regex: '\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b' + +- name: Stat pre ceph conf file + stat: + path: "{{ tripleo_cephadm_bootstrap_conf }}" + register: tripleo_cephadm_bootstrap_conf_stat + become: true + +- name: Fail if pre ceph conf file is missing + fail: + msg: "{{ tripleo_cephadm_bootstrap_conf }} does not exist according to stat" + when: + - tripleo_cephadm_bootstrap_conf_stat.stat.exists is not defined + +- name: Stat spec file on bootstrap node + stat: + path: "{{ item }}" + register: tripleo_cephadm_spec_files_stat + become: true + loop: + - "{{ tripleo_cephadm_spec }}" + +- name: Fail if spec file is missing + fail: + msg: "{{ item.invocation.module_args.path }} does not exist" + loop: "{{ tripleo_cephadm_spec_files_stat.results | list }}" + when: not item.stat.exists + +- name: Get ceph_cli + include_tasks: "../../tasks/ceph_cli.yaml" + vars: + mount_spec: true + +- name: Assert that ceph_cli contains expected parameters + assert: + that: + - tripleo_cephadm_ceph_cli | regex_search('^' + tripleo_cephadm_container_cli + ' run --rm') + - tripleo_cephadm_ceph_cli | regex_search(tripleo_cephadm_container_options) + - tripleo_cephadm_ceph_cli | regex_search(ceph_vol) + - tripleo_cephadm_ceph_cli | regex_search(spec_vol) + - tripleo_cephadm_ceph_cli | regex_search('--entrypoint ceph') + - tripleo_cephadm_ceph_cli | regex_search(image) + - tripleo_cephadm_ceph_cli | regex_search('--fsid ' + tripleo_cephadm_fsid) + - tripleo_cephadm_ceph_cli | regex_search('-c ' + tripleo_cephadm_conf) + - tripleo_cephadm_ceph_cli | regex_search('-k ' + tripleo_cephadm_admin_keyring) + vars: + ceph_vol: "--volume {{ tripleo_cephadm_config_home }}:{{ tripleo_cephadm_config_home }}:z" + spec_vol: "--volume {{ tripleo_cephadm_spec }}:{{ tripleo_cephadm_container_spec }}:z" + image: "{{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }}" + +- name: Confirm we can inlcude_vars the generated tripleo_ceph_client input file + include_vars: "{{ tripleo_ceph_client_vars }}" + delegate_to: localhost + +- name: Assert that the generated client vars are correct + assert: + that: + - tripleo_ceph_client_fsid == tripleo_cephadm_fsid + - external_cluster_mon_ips == ips + - keys[0].name == 'client.openstack' + - keys[0].key == 'AQATZBBgAAAAABAAUl/GZvcldk6G74AoZ2v2rg==' + - keys[0].caps.mgr == 'allow *' + - keys[0].caps.mon == 'profile rbd' + - keys[0].caps.osd == osd_profile + vars: + osd_profile: 'profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images' + ips: '[v2:172.16.11.241:3300/0,v1:172.16.11.241:6789/0],[v2:172.16.11.176:3300/0,v1:172.16.11.176:6789/0],[v2:172.16.11.82:3300/0,v1:172.16.11.82:6789/0]' diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml new file mode 100644 index 000000000..f385233d3 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml @@ -0,0 +1,55 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Stat spec file on bootstrap node + stat: + path: "{{ item }}" + register: tripleo_cephadm_spec_files_stat + become: true + loop: + - "{{ tripleo_cephadm_spec }}" + +- name: Fail if spec file is missing + fail: + msg: "{{ item.invocation.module_args.path }} does not exist" + loop: "{{ tripleo_cephadm_spec_files_stat.results | list }}" + when: not item.stat.exists + +- name: Get ceph_cli + include_tasks: ceph_cli.yaml + vars: + mount_spec: true + +- name: Get the ceph orchestrator status + command: "{{ tripleo_cephadm_ceph_cli }} orch status --format json" + register: ceph_orch_status + become: true + +- name: Fail if ceph orchestrator is not available + fail: + msg: "'ceph orch status' returned {{ ceph_orch_status.stdout | from_json }}" + when: + - not (ceph_orch_status.stdout | from_json).available + +- name: Apply spec + command: "{{ tripleo_cephadm_ceph_cli }} orch apply --in-file {{ tripleo_cephadm_container_spec }}" + register: tripleo_cephadm_apply_spec_out + become: true + +- name: Show results of spec apply + debug: + msg: "{{ tripleo_cephadm_apply_spec_out }}" + when: tripleo_cephadm_verbose diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/bootstrap.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/bootstrap.yaml new file mode 100644 index 000000000..9e54d2f0a --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/bootstrap.yaml @@ -0,0 +1,83 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Add spec to necessary file list when using spec to bootstrap + set_fact: + tripleo_cephadm_bootstrap_files: "{{ tripleo_cephadm_bootstrap_files + [ tripleo_cephadm_spec ] }}" + when: tripleo_cephadm_spec_on_bootstrap + +- name: Stat necessary files to bootstrap with cephadm + stat: + path: "{{ item }}" + register: tripleo_cephadm_bootstrap_files_stat + become: true + loop: "{{ tripleo_cephadm_bootstrap_files }}" + +- name: Fail if necessary files are missing + fail: + msg: "{{ item.invocation.module_args.path }} does not exist" + loop: "{{ tripleo_cephadm_bootstrap_files_stat.results | list }}" + when: not item.stat.exists + +- name: Stat pre ceph conf file in case we should bootrap with it + stat: + path: "{{ tripleo_cephadm_bootstrap_conf }}" + register: tripleo_cephadm_bootstrap_conf_stat + become: true + +# cephadm_ls should be registered by pre.yaml + +- name: Bootstrap Ceph if there are no running Ceph Daemons + block: + # items to add + # --registry-json + - name: Run cephadm bootstrap + shell: | + {{ tripleo_cephadm_bin }} \ + --image {{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }} \ + bootstrap \ + --skip-firewalld \ + --ssh-private-key /home/{{ tripleo_cephadm_ssh_user }}/.ssh/id_rsa \ + --ssh-public-key /home/{{ tripleo_cephadm_ssh_user }}/.ssh/id_rsa.pub \ + --ssh-user {{ tripleo_cephadm_ssh_user }} \ + --allow-fqdn-hostname \ + --output-keyring {{ tripleo_cephadm_admin_keyring }} \ + --output-config {{ tripleo_cephadm_conf }} \ + --fsid {{ tripleo_cephadm_fsid }} \ + {% if tripleo_cephadm_spec_on_bootstrap %}--apply-spec {{ tripleo_cephadm_spec }} \{% endif %} + {% if tripleo_cephadm_bootstrap_conf_stat.stat.exists %}--config {{ tripleo_cephadm_bootstrap_conf }} \{% endif %} + {% if not tripleo_cephadm_dashboard_enabled %} + --skip-monitoring-stack --skip-dashboard \ + {% endif %} + --mon-ip {{ tripleo_cephadm_first_mon_ip }} + register: cephadm_bootstrap + become: true + - name: Show results of bootstrap + debug: + msg: "{{ cephadm_bootstrap }}" + when: tripleo_cephadm_verbose + when: + - cephadm_ls.stdout == '[]' + tags: + - cephadm_bootstrap + +- name: If cephadm bootstrap was not run report the reason + debug: + msg: | + 'cephadm bootstrap' was not run because 'cephadm ls' + indicates that Ceph containers are already running. + when: + - cephadm_ls.stdout != '[]' diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml new file mode 100644 index 000000000..b3305f652 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml @@ -0,0 +1,27 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Set ceph CLI + set_fact: + tripleo_cephadm_ceph_cli: >- + {{ tripleo_cephadm_container_cli }} run --rm {{ tripleo_cephadm_container_options }} + --volume {{ tripleo_cephadm_config_home }}:/etc/ceph:z + {% if mount_spec|default(false) %} --volume {{ tripleo_cephadm_spec }}:{{ tripleo_cephadm_container_spec }}:z {% endif %} + --entrypoint {{ ceph_command | default('ceph') }} + {{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} + {% if ceph_command|default('ceph') == 'ceph' %} + --fsid {{ tripleo_cephadm_fsid }} -c {{ tripleo_cephadm_conf }} -k {{ tripleo_cephadm_admin_keyring }} + {% endif %} diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/export.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/export.yaml new file mode 100644 index 000000000..5bab2951e --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/export.yaml @@ -0,0 +1,80 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Creates a file which tripleo_ceph_client role can pass to include_vars +# The file will be saved in the path "{{ tripleo_ceph_client_vars }}" +# Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/ +# https://github.com/ceph/ceph-ansible/blob/master/library/ceph_key.py + +- name: Get ceph_cli + include_tasks: ceph_cli.yaml + +- name: Wait for the expected number of monitors to be running + include_tasks: wait_for_expected_num_mons.yaml + when: tripleo_cephadm_wait_for_mons + +- name: Run ceph mon dump to get all monitors + command: "{{ tripleo_cephadm_ceph_cli }} mon dump --format json" + register: ceph_mon_dump + become: true + tags: + - cephadm_mon_dump + +- name: Extract mons_json + set_fact: + tripleo_cephadm_mons_json: "{{ (ceph_mon_dump.stdout | from_json).mons | + map(attribute='public_addrs') | + map(attribute='addrvec') | + list }}" + +- name: Build mons_list + set_fact: + tripleo_cephadm_mons_list: "{{ tripleo_cephadm_mons_list | default([]) + + [ '[' ~ + item[0].type ~ ':' ~ item[0].addr ~ '/' ~ item[0].nonce + ~ ',' ~ + item[1].type ~ ':' ~ item[1].addr ~ '/' ~ item[1].nonce + ~ ']' + ] }}" + loop: "{{ tripleo_cephadm_mons_json }}" + +- name: Set external_cluster_mon_ips from mons_list + set_fact: + external_cluster_mon_ips: "{{ tripleo_cephadm_mons_list | join(',') }}" + +- name: Extract keys + ceph_key: + name: "{{ item.name }}" + state: info + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + register: tripleo_cephadm_client_keys + become: true + loop: "{{ tripleo_cephadm_keys }}" + when: + - tripleo_cephadm_keys is defined + - tripleo_cephadm_keys | length > 0 + tags: + - cephadm_extract_keys + +- name: Save tripleo_ceph_client_vars file + template: + src: templates/ceph_client.yaml.j2 + dest: "{{ tripleo_ceph_client_vars }}" + mode: 0644 + force: true + delegate_to: localhost diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/keys.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/keys.yaml new file mode 100644 index 000000000..7da27967c --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/keys.yaml @@ -0,0 +1,38 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/ +# https://github.com/ceph/ceph-ansible/blob/master/library/ceph_key.py + +- name: create cephx key(s) + ceph_key: + import_key: true + name: "{{ item.name }}" + caps: "{{ item.caps }}" + mode: "{{ item.mode }}" + secret: "{{ item.key | default('') }}" + cluster: "{{ tripleo_cephadm_cluster }}" + dest: "{{ tripleo_cephadm_config_home }}" + owner: "{{ tripleo_cephadm_uid }}" + group: "{{ tripleo_cephadm_uid }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + become: true + loop: "{{ tripleo_cephadm_keys }}" + when: + - tripleo_cephadm_keys is defined + - tripleo_cephadm_keys | length > 0 diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/main.yml b/tripleo_ansible/roles/tripleo_cephadm/tasks/main.yml new file mode 100644 index 000000000..bafb2036c --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/main.yml @@ -0,0 +1,2 @@ +--- +# tasks file for tripleo_cephadm diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/pools.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/pools.yaml new file mode 100644 index 000000000..4fe629bb0 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/pools.yaml @@ -0,0 +1,41 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/ +# https://github.com/ceph/ceph-ansible/blob/master/library/ceph_pool.py + +- name: Create pool(s) + ceph_pool: + name: "{{ item.name }}" + cluster: "{{ tripleo_cephadm_cluster }}" + pg_num: "{{ item.pg_num | default(omit) }}" + pgp_num: "{{ item.pgp_num | default(omit) }}" + pg_autoscale_mode: "{{ item.pg_autoscale_mode | default(omit) }}" + target_size_ratio: "{{ item.target_size_ratio | default(omit) }}" + size: "{{ item.size | default(omit) }}" + min_size: "{{ item.min_size | default(omit) }}" + pool_type: "{{ item.type | default('replicated') }}" + rule_name: "{{ item.rule_name | default(omit) }}" + erasure_profile: "{{ item.erasure_profile | default(omit) }}" + application: "{{ item.application | default(omit) }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + become: true + with_items: "{{ tripleo_cephadm_pools }}" + when: + - tripleo_cephadm_pools is defined + - tripleo_cephadm_pools | length > 0 diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/pre.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/pre.yaml new file mode 100644 index 000000000..5fe166682 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/pre.yaml @@ -0,0 +1,113 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Install cephadm package + become: true + package: + name: cephadm + state: latest + when: + - tripleo_cephadm_predeployed | bool + +- name: Stat cephadm file + stat: + path: "{{ tripleo_cephadm_bin }}" + register: stat_cephadm + become: true + ignore_errors: true + +- name: Fail if cephadm is not available + fail: + msg: "{{ tripleo_cephadm_bin }} does not exist" + when: not stat_cephadm.stat.exists + +- name: List Ceph daemon instances on this host + shell: "{{ tripleo_cephadm_bin }} ls --no-detail" + register: cephadm_ls + become: true + +- name: Ensure tripleo_cephadm_fsid variable is set if none was provided + block: + - name: Set list of found FSIDs + set_fact: + tripleo_cephadm_fsid_list: "{{ cephadm_ls.stdout | from_json | map(attribute='fsid') | sort | unique }}" + when: + - cephadm_ls.stdout is defined + - cephadm_ls.stdout != '[]' + - name: Fail if >1 FSID was discovered + fail: + msg: | + Multiple FSIDs were found. This Ansible role does not + support management of multiple Ceph clusters on one host. + when: tripleo_cephadm_fsid_list | length > 1 + - name: Set FSID to the discovered value + set_fact: + tripleo_cephadm_fsid: "{{ tripleo_cephadm_fsid_list[0] }}" + when: tripleo_cephadm_fsid_list | length == 1 + - name: Set random tripleo_cephadm_fsid if no running ceph containers were found + set_fact: + tripleo_cephadm_fsid: "{{ 99999999 | random | to_uuid | lower }}" + when: tripleo_cephadm_fsid_list | length == 0 + when: tripleo_cephadm_fsid is not defined or (tripleo_cephadm_fsid is defined and tripleo_cephadm_fsid | length == 0) + +- name: Set first monitor IP if it was not passed + set_fact: + tripleo_cephadm_first_mon_ip: "{{ ansible_host }}" + when: tripleo_cephadm_first_mon_ip is not defined or (tripleo_cephadm_first_mon_ip is defined and tripleo_cephadm_first_mon_ip | length == 0) + +- name: Ensure tripleo_cephadm_config_home (e.g. /etc/ceph) exists + file: + path: "{{ tripleo_cephadm_config_home }}" + state: directory + become: true + +- name: Ensure specs directory exists + file: + path: "/home/{{ tripleo_cephadm_ssh_user }}/specs" + owner: "{{ tripleo_cephadm_ssh_user }}" + group: "{{ tripleo_cephadm_ssh_user }}" + mode: '0755' + state: directory + become: true + +- name: Stat spec file on ansible host + stat: + path: "{{ tripleo_cephadm_spec_ansible_host }}" + register: tripleo_cephadm_spec_stat_host + delegate_to: localhost + become: true + +- name: push tripleo_cephadm_spec to bootstrap node if spec file exsits + copy: + src: "{{ tripleo_cephadm_spec_ansible_host }}" + dest: "{{ tripleo_cephadm_spec }}" + owner: "{{ tripleo_cephadm_ssh_user }}" + group: "{{ tripleo_cephadm_ssh_user }}" + mode: '0644' + become: true + when: tripleo_cephadm_spec_stat_host.stat.exists + +- name: generate pre ceph.conf configuration file for initial bootstrap + action: config_template + args: + src: "ceph.conf.j2" + dest: "{{ tripleo_cephadm_bootstrap_conf }}" + owner: "{{ tripleo_cephadm_uid }}" + group: "{{ tripleo_cephadm_uid }}" + mode: "0644" + config_overrides: "{{ tripleo_cephadm_conf_overrides }}" + config_type: ini + become: true diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/wait_for_expected_num_mons.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/wait_for_expected_num_mons.yaml new file mode 100644 index 000000000..865f4eb7e --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/wait_for_expected_num_mons.yaml @@ -0,0 +1,44 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Get the expected number of mons + block: + - name: Read the spec file + set_fact: + tripleo_cephadm_spec_content: "{{ lookup('file', tripleo_cephadm_spec_ansible_host) }}" + - name: Parse each yaml document in the spec file looking for the list of mons + set_fact: + tripleo_cephadm_num_mons_expected: "{{ item.placement.hosts | list | length | int }}" + loop: "{{ tripleo_cephadm_spec_content | from_yaml_all | list }}" + when: + - item | length > 0 + - item.service_type is defined + - item.service_type == 'mon' + - item.placement is defined + - item.placement.hosts is defined + when: + - tripleo_cephadm_num_mons_expected is not defined + - tripleo_cephadm_spec_ansible_host is defined + - tripleo_cephadm_spec_ansible_host | length > 0 + +- name: Wait for expected number of mons to be running + shell: "{{ tripleo_cephadm_ceph_cli }} status --format json | jq .monmap.num_mons" + register: ceph_status + become: true + until: (ceph_status.stdout | int) >= ((tripleo_cephadm_num_mons_expected | int) | default(1)) + retries: "{{ tripleo_cephadm_wait_for_mons_retries }}" + delay: "{{ tripleo_cephadm_wait_for_mons_delay }}" + ignore_errors: "{{ tripleo_cephadm_wait_for_mons_ignore_errors }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/templates/ceph.conf.j2 b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph.conf.j2 new file mode 100644 index 000000000..3f443d3b1 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph.conf.j2 @@ -0,0 +1,13 @@ +#jinja2: trim_blocks: "true", lstrip_blocks: "true" +# {{ ansible_managed }} +# Generated by tripleo_cephadm for initial bootstrap of first Ceph Mon + +[global] +fsid = {{ tripleo_cephadm_fsid }} +mon host = {{ tripleo_cephadm_first_mon_ip }} +{% if public_network is defined %} +public network = {{ public_network | regex_replace(' ', '') }} +{% endif %} +{% if cluster_network is defined %} +cluster network = {{ cluster_network | regex_replace(' ', '') }} +{% endif %} diff --git a/tripleo_ansible/roles/tripleo_cephadm/templates/ceph_client.yaml.j2 b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph_client.yaml.j2 new file mode 100644 index 000000000..576d31acb --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph_client.yaml.j2 @@ -0,0 +1,14 @@ +--- +tripleo_ceph_client_fsid: {{ tripleo_cephadm_fsid }} +external_cluster_mon_ips: "{{ external_cluster_mon_ips }}" +keys: +{% for ceph_key_cmd in tripleo_cephadm_client_keys.results %} +{% for cephx in (ceph_key_cmd.stdout | from_json) %} +- name: {{ cephx.entity }} + key: {{ cephx.key }} + caps: +{% for key, value in cephx.caps.items() %} + {{ key }}: {{ value }} +{% endfor %} +{% endfor %} +{% endfor %} diff --git a/zuul.d/molecule.yaml b/zuul.d/molecule.yaml index e0554c813..91d5a3fa5 100644 --- a/zuul.d/molecule.yaml +++ b/zuul.d/molecule.yaml @@ -14,6 +14,7 @@ - tripleo-ansible-centos-8-molecule-tripleo_cellv2 - tripleo-ansible-centos-8-molecule-tripleo_ceph_client - tripleo-ansible-centos-8-molecule-tripleo_ceph_run_ansible + - tripleo-ansible-centos-8-molecule-tripleo_cephadm - tripleo-ansible-centos-8-molecule-tripleo_clients_install - tripleo-ansible-centos-8-molecule-tripleo_config - tripleo-ansible-centos-8-molecule-tripleo_container_image_build @@ -52,11 +53,11 @@ - tripleo-ansible-centos-8-molecule-tripleo_systemd_wrapper - tripleo-ansible-centos-8-molecule-tripleo_timezone - tripleo-ansible-centos-8-molecule-tripleo_transfer + - tripleo-ansible-centos-8-molecule-tripleo_update_trusted_cas - tripleo-ansible-centos-8-molecule-tripleo_upgrade_hiera - tripleo-ansible-centos-8-molecule-tripleo_validations_package - tripleo-ansible-centos-8-molecule-tuned - tripleo-ansible-centos-8-role-addition - - tripleo-ansible-centos-8-molecule-tripleo_update_trusted_cas gate: jobs: - tripleo-ansible-centos-8-molecule-aide @@ -69,6 +70,7 @@ - tripleo-ansible-centos-8-molecule-tripleo_cellv2 - tripleo-ansible-centos-8-molecule-tripleo_ceph_client - tripleo-ansible-centos-8-molecule-tripleo_ceph_run_ansible + - tripleo-ansible-centos-8-molecule-tripleo_cephadm - tripleo-ansible-centos-8-molecule-tripleo_clients_install - tripleo-ansible-centos-8-molecule-tripleo_config - tripleo-ansible-centos-8-molecule-tripleo_container_image_build @@ -107,12 +109,11 @@ - tripleo-ansible-centos-8-molecule-tripleo_systemd_wrapper - tripleo-ansible-centos-8-molecule-tripleo_timezone - tripleo-ansible-centos-8-molecule-tripleo_transfer + - tripleo-ansible-centos-8-molecule-tripleo_update_trusted_cas - tripleo-ansible-centos-8-molecule-tripleo_upgrade_hiera - tripleo-ansible-centos-8-molecule-tripleo_validations_package - tripleo-ansible-centos-8-molecule-tuned - tripleo-ansible-centos-8-role-addition - - tripleo-ansible-centos-8-molecule-tripleo_update_trusted_cas - name: tripleo-ansible-molecule-jobs periodic-weekly: jobs: @@ -126,6 +127,7 @@ - tripleo-ansible-centos-8-molecule-tripleo_cellv2 - tripleo-ansible-centos-8-molecule-tripleo_ceph_client - tripleo-ansible-centos-8-molecule-tripleo_ceph_run_ansible + - tripleo-ansible-centos-8-molecule-tripleo_cephadm - tripleo-ansible-centos-8-molecule-tripleo_clients_install - tripleo-ansible-centos-8-molecule-tripleo_config - tripleo-ansible-centos-8-molecule-tripleo_container_image_build @@ -238,6 +240,14 @@ vars: tripleo_job_ansible_args: -v --skip-tags=run_uuid_ansible,run_ceph_ansible tripleo_role_name: tripleo_ceph_run_ansible +- job: + files: + - ^tripleo_ansible/roles/tripleo_cephadm/.* + name: tripleo-ansible-centos-8-molecule-tripleo_cephadm + parent: tripleo-ansible-centos-8-base + vars: + tripleo_job_ansible_args: --skip-tags=cephadm_bootstrap,cephadm_mon_dump,cephadm_extract_keys + tripleo_role_name: tripleo_cephadm - job: files: - ^tripleo_ansible/roles/tripleo_clients_install/.* @@ -518,6 +528,13 @@ parent: tripleo-ansible-centos-8-base vars: tripleo_role_name: tripleo_transfer +- job: + files: + - ^tripleo_ansible/roles/tripleo_update_trusted_cas/.* + name: tripleo-ansible-centos-8-molecule-tripleo_update_trusted_cas + parent: tripleo-ansible-centos-8-base + vars: + tox_envlist: mol-tripleo_update_trusted_cas - job: files: - ^tripleo_ansible/roles/tripleo_upgrade_hiera/.* @@ -554,11 +571,3 @@ timeout: 1800 vars: tox_envlist: role-addition - -- job: - files: - - ^tripleo_ansible/roles/tripleo_update_trusted_cas/.* - name: tripleo-ansible-centos-8-molecule-tripleo_update_trusted_cas - parent: tripleo-ansible-centos-8-base - vars: - tox_envlist: mol-tripleo_update_trusted_cas