From 158481b8ea23983c47ee9174b5ebd1e59d02ae43 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Mon, 1 Feb 2021 12:08:28 +0100 Subject: [PATCH] Deploy RGW, MDS, Ganesha and Ceph Dashboard using cephadm This change adds the mkspec module which is able to build a daemon specific spec and apply it against the deployed Ceph cluster. In addition, multiple tasks have been added with the purpose of integrating all the scenario004 services and properly manage the cephadm playbook flow according to the enabled services. Change-Id: I35e57abddc64310a6422174fe191bd328588d7cd --- .ansible-lint | 3 + doc/source/modules/modules-ceph_spec.rst | 14 + .../modules/ceph_dashboard_user.py | 265 ++++++++++++++++++ .../ansible_plugins/modules/ceph_fs.py | 256 +++++++++++++++++ .../ansible_plugins/modules/ceph_mkspec.py | 248 ++++++++++++++++ tripleo_ansible/playbooks/cephadm.yml | 30 ++ .../roles/tripleo_cephadm/defaults/main.yml | 7 + .../tripleo_cephadm/files/ceph_spec.yaml | 6 + .../tripleo_cephadm/tasks/apply_spec.yaml | 7 + .../tripleo_cephadm/tasks/bootstrap.yaml | 2 - .../tasks/config_container_images.yaml | 57 ++++ .../configure_dashboard_backends.yml | 37 +++ .../tasks/dashboard/dashboard.yaml | 108 +++++++ .../tasks/dashboard/grafana.yaml | 55 ++++ .../roles/tripleo_cephadm/tasks/ganesha.yaml | 42 +++ .../roles/tripleo_cephadm/tasks/mds.yaml | 77 +++++ .../tripleo_cephadm/tasks/monitoring.yaml | 64 +++++ .../roles/tripleo_cephadm/tasks/post.yaml | 48 ++++ .../roles/tripleo_cephadm/tasks/rgw.yaml | 48 ++++ .../tasks/wait_for_expected_num_osds.yaml | 24 ++ .../tripleo_cephadm/templates/ceph.conf.j2 | 26 ++ .../tripleo_cephadm/templates/grafana.json.j2 | 39 +++ .../tripleo_run_cephadm/tasks/prepare.yml | 3 +- .../tests/modules/test_ceph_mkspec.py | 55 ++++ 24 files changed, 1518 insertions(+), 3 deletions(-) create mode 100644 doc/source/modules/modules-ceph_spec.rst create mode 100644 tripleo_ansible/ansible_plugins/modules/ceph_dashboard_user.py create mode 100644 tripleo_ansible/ansible_plugins/modules/ceph_fs.py create mode 100644 tripleo_ansible/ansible_plugins/modules/ceph_mkspec.py create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/config_container_images.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/configure_dashboard_backends.yml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/dashboard.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/grafana.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/mds.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/monitoring.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/post.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/rgw.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/wait_for_expected_num_osds.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/templates/grafana.json.j2 create mode 100644 tripleo_ansible/tests/modules/test_ceph_mkspec.py diff --git a/.ansible-lint b/.ansible-lint index 7bc09d710..143e8aa30 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -14,8 +14,11 @@ mock_roles: mock_modules: - baremetal_nodes_validate - baremetal_register_or_update_nodes + - ceph_dashboard_user - ceph_key + - ceph_fs - ceph_pool + - ceph_mkspec - config_template - container_startup_config - lvm2_physical_devices_facts diff --git a/doc/source/modules/modules-ceph_spec.rst b/doc/source/modules/modules-ceph_spec.rst new file mode 100644 index 000000000..605a4a10d --- /dev/null +++ b/doc/source/modules/modules-ceph_spec.rst @@ -0,0 +1,14 @@ +============================ +Module - ceph_mkspec +============================ + + +This module provides for the following ansible plugin: + + * ceph_mkspec + + +.. ansibleautoplugin:: + :module: tripleo_ansible/ansible_plugins/modules/ceph_mkspec.py + :documentation: true + :examples: true diff --git a/tripleo_ansible/ansible_plugins/modules/ceph_dashboard_user.py b/tripleo_ansible/ansible_plugins/modules/ceph_dashboard_user.py new file mode 100644 index 000000000..9e1de74d2 --- /dev/null +++ b/tripleo_ansible/ansible_plugins/modules/ceph_dashboard_user.py @@ -0,0 +1,265 @@ +# Copyright 2020, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from ansible.module_utils.basic import AnsibleModule +try: + from ansible.module_utils.ca_common import generate_ceph_cmd, \ + is_containerized, \ + exec_command, \ + exit_module +except ImportError: + from tripleo_ansible.ansible_plugins.module_utils.ca_common import generate_ceph_cmd, is_containerized, exec_command, exit_module + +import datetime +import json +import yaml + + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = """ +--- +module: ceph_dashboard_user + +short_description: Manage Ceph Dashboard User + +version_added: "2.8" + +description: + - Manage Ceph Dashboard user(s) creation, deletion and updates. +options: + cluster: + description: + - The ceph cluster name. + required: false + default: ceph + type: str + name: + description: + - name of the Ceph Dashboard user. + required: true + type: str + state: + description: + If 'present' is used, the module creates a user if it doesn't + exist or update it if it already exists. + If 'absent' is used, the module will simply delete the user. + If 'info' is used, the module will return all details about the + existing user (json formatted). + required: false + choices: ['present', 'absent', 'info'] + default: present + type: str + password: + description: + - password of the Ceph Dashboard user. + required: false + type: str + roles: + description: + - roles of the Ceph Dashboard user. + required: false + choices: ['administrator', 'read-only', 'block-manager', 'rgw-manager', 'cluster-manager', 'pool-manager', 'cephfs-manager'] + type: list + default: [] + +author: + - Dimitri Savineau +""" + +EXAMPLES = ''' +- name: create a Ceph Dashboard user + ceph_dashboard_user: + name: foo + password: bar + +- name: create a read-only/block-manager Ceph Dashboard user + ceph_dashboard_user: + name: foo + password: bar + roles: + - 'read-only' + - 'block-manager' + +- name: create a Ceph Dashboard admin user + ceph_dashboard_user: + name: foo + password: bar + roles: ['administrator'] + +- name: get a Ceph Dashboard user information + ceph_dashboard_user: + name: foo + state: info + +- name: delete a Ceph Dashboard user + ceph_dashboard_user: + name: foo + state: absent +''' + +RETURN = '''# ''' + + +def create_user(module, container_image=None): + ''' + Create a new user + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['ac-user-create', '-i', '-', name] + + cmd = generate_ceph_cmd(sub_cmd=['dashboard'], args=args, spec_path=None, cluster=cluster, container_image=container_image, interactive=True) + + return cmd + + +def set_roles(module, container_image=None): + ''' + Set user roles + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + roles = module.params.get('roles') + + args = ['ac-user-set-roles', name] + + args.extend(roles) + + cmd = generate_ceph_cmd(sub_cmd=['dashboard'], args=args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def set_password(module, container_image=None): + ''' + Set user password + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['ac-user-set-password', '-i', '-', name] + + cmd = generate_ceph_cmd(sub_cmd=['dashboard'], args=args, spec_path=None, cluster=cluster, container_image=container_image, interactive=True) + + return cmd + + +def get_user(module, container_image=None): + ''' + Get existing user + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['ac-user-show', name, '--format=json'] + + cmd = generate_ceph_cmd(sub_cmd=['dashboard'], args=args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def remove_user(module, container_image=None): + ''' + Remove a user + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['ac-user-delete', name] + + cmd = generate_ceph_cmd(sub_cmd=['dashboard'], args=args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def run_module(): + module = AnsibleModule( + argument_spec=yaml.safe_load(DOCUMENTATION)['options'], + supports_check_mode=True, + required_if=[['state', 'present', ['password']]] + ) + + # Gather module parameters in variables + name = module.params.get('name') + state = module.params.get('state') + roles = module.params.get('roles') + password = module.params.get('password') + + if module.check_mode: + module.exit_json( + changed=False, + stdout='', + stderr='', + rc=0, + start='', + end='', + delta='', + ) + + startd = datetime.datetime.now() + changed = False + + # will return either the image name or None + container_image = is_containerized() + + if state == "present": + rc, cmd, out, err = exec_command(module, get_user(module, container_image=container_image)) + if rc == 0: + user = json.loads(out) + user['roles'].sort() + roles.sort() + if user['roles'] != roles: + rc, cmd, out, err = exec_command(module, set_roles(module, container_image=container_image)) + changed = True + rc, cmd, out, err = exec_command(module, set_password(module, container_image=container_image), stdin=password) + else: + rc, cmd, out, err = exec_command(module, create_user(module, container_image=container_image), stdin=password) + rc, cmd, out, err = exec_command(module, set_roles(module, container_image=container_image)) + changed = True + + elif state == "absent": + rc, cmd, out, err = exec_command(module, get_user(module, container_image=container_image)) + if rc == 0: + rc, cmd, out, err = exec_command(module, remove_user(module, container_image=container_image)) + changed = True + else: + rc = 0 + out = "Dashboard User {} doesn't exist".format(name) + + elif state == "info": + rc, cmd, out, err = exec_command(module, get_user(module, container_image=container_image)) + + exit_module(module=module, out=out, rc=rc, cmd=cmd, err=err, startd=startd, changed=changed) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/tripleo_ansible/ansible_plugins/modules/ceph_fs.py b/tripleo_ansible/ansible_plugins/modules/ceph_fs.py new file mode 100644 index 000000000..4a9694efb --- /dev/null +++ b/tripleo_ansible/ansible_plugins/modules/ceph_fs.py @@ -0,0 +1,256 @@ +# Copyright 2020, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Included from: https://github.com/ceph/ceph-ansible/blob/master/library/ceph_fs.py + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from ansible.module_utils.basic import AnsibleModule +try: + from ansible.module_utils.ca_common import is_containerized, \ + exec_command, \ + generate_ceph_cmd, \ + exit_module +except ImportError: + from tripleo_ansible.ansible_plugins.module_utils.ca_common import is_containerized, \ + exec_command, \ + generate_ceph_cmd, \ + exit_module + +import datetime +import json +import yaml + + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = """ +--- +module: ceph_fs + +short_description: Manage Ceph File System + +version_added: "2.8" + +description: + - Manage Ceph File System(s) creation, deletion and updates. +options: + cluster: + description: + - The ceph cluster name. + required: false + default: ceph + name: + description: + - name of the Ceph File System. + required: true + state: + description: + If 'present' is used, the module creates a filesystem if it + doesn't exist or update it if it already exists. + If 'absent' is used, the module will simply delete the filesystem. + If 'info' is used, the module will return all details about the + existing filesystem (json formatted). + required: false + choices: ['present', 'absent', 'info'] + default: present + data: + description: + - name of the data pool. + required: false + metadata: + description: + - name of the metadata pool. + required: false + max_mds: + description: + - name of the max_mds attribute. + required: false + +author: + - Dimitri Savineau +""" + +EXAMPLES = ''' +- name: create a Ceph File System + ceph_fs: + name: foo + data: bar_data + metadata: bar_metadata + max_mds: 2 + +- name: get a Ceph File System information + ceph_fs: + name: foo + state: info + +- name: delete a Ceph File System + ceph_fs: + name: foo + state: absent +''' + +RETURN = '''# ''' + + +def create_fs(module, container_image=None): + ''' + Create a new fs + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + data = module.params.get('data') + metadata = module.params.get('metadata') + + args = ['new', name, metadata, data] + + cmd = generate_ceph_cmd(sub_cmd=['fs'], args=args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def get_fs(module, container_image=None): + ''' + Get existing fs + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['get', name, '--format=json'] + + cmd = generate_ceph_cmd(sub_cmd=['fs'], args=args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def remove_fs(module, container_image=None): + ''' + Remove a fs + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['rm', name, '--yes-i-really-mean-it'] + + cmd = generate_ceph_cmd(sub_cmd=['fs'], args=args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def fail_fs(module, container_image=None): + ''' + Fail a fs + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['fail', name] + + cmd = generate_ceph_cmd(sub_cmd=['fs'], args=args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def set_fs(module, container_image=None): + ''' + Set parameter to a fs + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + max_mds = module.params.get('max_mds') + + args = ['set', name, 'max_mds', str(max_mds)] + + cmd = generate_ceph_cmd(sub_cmd=['fs'], args=args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def run_module(): + + module = AnsibleModule( + argument_spec=yaml.safe_load(DOCUMENTATION)['options'], + supports_check_mode=True, + required_if=[['state', 'present', ['data', 'metadata']]], + ) + + # Gather module parameters in variables + name = module.params.get('name') + state = module.params.get('state') + max_mds = module.params.get('max_mds') + + if module.check_mode: + module.exit_json( + changed=False, + stdout='', + stderr='', + rc=0, + start='', + end='', + delta='', + ) + + startd = datetime.datetime.now() + changed = False + + # will return either the image name or None + container_image = is_containerized() + + if state == "present": + rc, cmd, out, err = exec_command(module, get_fs(module, container_image=container_image)) + if rc == 0: + fs = json.loads(out) + if max_mds and fs["mdsmap"]["max_mds"] != max_mds: + rc, cmd, out, err = exec_command(module, set_fs(module, container_image=container_image)) + if rc == 0: + changed = True + else: + rc, cmd, out, err = exec_command(module, create_fs(module, container_image=container_image)) + if max_mds and max_mds > 1: + exec_command(module, set_fs(module, container_image=container_image)) + if rc == 0: + changed = True + + elif state == "absent": + rc, cmd, out, err = exec_command(module, get_fs(module, container_image=container_image)) + if rc == 0: + exec_command(module, fail_fs(module, container_image=container_image)) + rc, cmd, out, err = exec_command(module, remove_fs(module, container_image=container_image)) + if rc == 0: + changed = True + else: + rc = 0 + out = "Ceph File System {} doesn't exist".format(name) + + elif state == "info": + rc, cmd, out, err = exec_command(module, get_fs(module, container_image=container_image)) + + exit_module(module=module, out=out, rc=rc, cmd=cmd, err=err, startd=startd, changed=changed) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/tripleo_ansible/ansible_plugins/modules/ceph_mkspec.py b/tripleo_ansible/ansible_plugins/modules/ceph_mkspec.py new file mode 100644 index 000000000..acece912f --- /dev/null +++ b/tripleo_ansible/ansible_plugins/modules/ceph_mkspec.py @@ -0,0 +1,248 @@ +# Copyright 2020, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function +from ansible.module_utils.basic import AnsibleModule +try: + from ansible.module_utils.ca_common import is_containerized, \ + exec_command, generate_ceph_cmd, exit_module +except ImportError: + from tripleo_ansible.ansible_plugins.module_utils.ca_common import is_containerized, \ + exec_command, generate_ceph_cmd, exit_module +try: + from ansible.module_utils import ceph_spec +except ImportError: + from tripleo_ansible.ansible_plugins.module_utils import ceph_spec + +import datetime +import json +import os +import stat +import time +import yaml + + +__metaclass__ = type + + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = """ +--- +module: ceph_mkspec + +short_description: Build cephadm spec + +version_added: "2.8" + +description: + - Build a cephadm compatible spec, rendering the daemon specified +options: + cluster: + description: + - The ceph cluster name. + required: false + default: ceph + type: str + service_type: + description: + - The Ceph daemon that is going to be applied + required: true + type: str + choices: ['host', 'mon', 'osd', 'mds', 'rgw', 'nfs', 'node-exporter', 'prometheus', 'alertmanager', 'grafana'] + service_id: + description: + - The ID associated to the Ceph daemon + required: false + type: str + service_name: + description: + - The name of the Ceph Daemon + required: false + type: str + hosts: + description: + - The host list where the daemon is going to be applied + required: false + type: list + host_pattern: + description: + - The host pattern where the daemon is going to be applied + required: false + type: str + labels: + description: + - The list of labels used to apply the daemon on the Ceph custer + nodes. + required: false + type: list + spec: + description: + - The spec definition of the daemon + type: dict + required: false + extra: + description: + - The extra top level key definition for the daemon + type: dict + required: false + render_path: + description: + - Where the spec will be rendered + type: str + required: false + apply: + description: + - If truem the spec rendered will be applied by the orchestrator + type: bool + required: false + default: false +""" + +EXAMPLES = ''' +- name: create the Ceph MDS daemon spec + ceph_spec: + service_type: mds + service_id: mds + service_name: mds + render_path: '/home/ceph-admin/specs' + hosts: + - host1 + - host2 + - hostN + apply: true + +- name: create the Ceph MDS daemon spec + ceph_spec: + service_type: mds + service_id: mds + service_name: mds + host_pattern: "*mon*" + apply: false + +- name: create the Ceph MDS daemon spec + ceph_spec: + service_type: mds + service_id: mds + service_name: mds + render_path: '/home/ceph-admin/specs' + labels: + - "controller" + apply: true +''' + +RETURN = '''# ''' + +ALLOWED_DAEMONS = ['host', 'mon', 'mgr', 'mds', 'nfs', 'osd', 'rgw', 'grafana', + 'crash', 'prometheus', 'alertmanager', 'node-exporter'] + + +def generate_orch_cli(cluster, spec_path, container_image): + + args = ['apply', '--in-file', spec_path] + + cmd = generate_ceph_cmd(sub_cmd=['orch'], args=args, + spec_path=spec_path, cluster=cluster, + container_image=container_image) + return cmd + + +def render(path, content): + if len(content) > 0: + if path is not None and len(path) > 0: + with open(path, 'w') as f: + f.write('---\n') + f.write(yaml.dump(content, indent=2)) + else: + print('Nothing to dump!') + + +def run_module(): + + module = AnsibleModule( + argument_spec=yaml.safe_load(DOCUMENTATION)['options'], + supports_check_mode=True, + required_if=[['apply', True, ['render_path']]], + ) + + # Gather module parameters in variables + cluster = module.params.get('cluster') + service_type = module.params.get('service_type') + service_id = module.params.get('service_type') + service_name = module.params.get('service_name') + hosts = module.params.get('hosts') + host_pattern = module.params.get('host_pattern') + labels = module.params.get('labels') + spec = module.params.get('spec') + extra = module.params.get('extra') + apply = module.params.get('apply') + render_path = module.params.get('render_path') + + if module.check_mode: + module.exit_json( + changed=False, + stdout='', + stderr='', + rc=0, + start='', + end='', + delta='', + ) + + startd = datetime.datetime.now() + changed = False + + # PROCESSING PARAMETERS + if service_id is None: + service_id = service_type + if service_name is None: + service_name = "{}.{}".format(service_type, service_id) + + # no spec is provided + if spec is None: + spec = {} + + # no spec is provided + if extra is None: + extra = {} + + # no labels are defined + if labels is None: + labels = [] + + d = ceph_spec.CephDaemonSpec(service_type, service_id, service_name, + hosts, host_pattern, spec, labels, **extra) + + if apply: + container_image = is_containerized() + render('{}/{}'.format(render_path, service_type), d.make_daemon_spec()) + cmd = generate_orch_cli(cluster, '{}/{}'.format(render_path, service_type), container_image) + rc, cmd, out, err = exec_command(module, cmd) + # module.exit_json(changed=True, result=cmd) + exit_module(module=module, out=out, rc=rc, cmd=cmd, err=err, startd=startd, changed=changed) + else: + # render the dict as the output of the module + module.exit_json(changed=True, result=d.make_daemon_spec()) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/tripleo_ansible/playbooks/cephadm.yml b/tripleo_ansible/playbooks/cephadm.yml index c4d53fd86..5859f2548 100644 --- a/tripleo_ansible/playbooks/cephadm.yml +++ b/tripleo_ansible/playbooks/cephadm.yml @@ -37,6 +37,31 @@ name: tripleo_cephadm tasks_from: pools + - name: Config RGW + import_role: + name: tripleo_cephadm + tasks_from: rgw + + - name: Config MDS and Ganesha daemons + block: + - name: Config MDS + import_role: + name: tripleo_cephadm + tasks_from: mds + + - name: Config Ganesha + import_role: + name: tripleo_cephadm + tasks_from: ganesha + when: + - cephfs_data_pool is defined + - cephfs_metadata_pool is defined + + - name: Configure Monitoring Stack + import_role: + name: tripleo_cephadm + tasks_from: monitoring + - name: Create Keys import_role: name: tripleo_cephadm @@ -46,3 +71,8 @@ import_role: name: tripleo_cephadm tasks_from: export + + - name: Show the Ceph cluster status + import_role: + name: tripleo_cephadm + tasks_from: post diff --git a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml index 2dd69cdb9..cf1eb4d8f 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml +++ b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml @@ -15,6 +15,7 @@ tripleo_cephadm_admin_keyring: "{{ tripleo_cephadm_config_home }}/{{ tripleo_cep tripleo_cephadm_conf: "{{ tripleo_cephadm_config_home }}/{{ tripleo_cephadm_cluster }}.conf" tripleo_cephadm_bootstrap_conf: "/home/{{ tripleo_cephadm_ssh_user }}/bootstrap_{{ tripleo_cephadm_cluster }}.conf" tripleo_cephadm_spec: "/home/{{ tripleo_cephadm_ssh_user }}/specs/ceph_spec.yaml" +tripleo_cephadm_spec_home: "/home/{{ tripleo_cephadm_ssh_user }}/specs" tripleo_cephadm_container_spec: /home/ceph_spec.yaml tripleo_cephadm_spec_ansible_host: "{{ role_path }}/files/ceph_spec.yaml" tripleo_cephadm_bootstrap_files: @@ -29,8 +30,14 @@ tripleo_cephadm_wait_for_mons: true tripleo_cephadm_wait_for_mons_retries: 10 tripleo_cephadm_wait_for_mons_delay: 20 tripleo_cephadm_wait_for_mons_ignore_errors: false +tripleo_cephadm_wait_for_osds: true +tripleo_cephadm_wait_for_osds_retries: 10 +tripleo_cephadm_wait_for_osds_delay: 20 +tripleo_cephadm_wait_for_osds_ignore_errors: false +tripleo_cephadm_num_osd_expected: 1 tripleo_cephadm_predeployed: true tripleo_cephadm_conf_overrides: {} tripleo_cephadm_fsid_list: [] +tripleo_cephadm_fqdn: false # todo(fultonj) add is_hci boolean for target memory # https://lists.ceph.io/hyperkitty/list/dev@ceph.io/thread/Z77XO23JPXDNHKM7IG6UN4URYKA6L7VH/ diff --git a/tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml b/tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml index bcd6585fa..46cf427d8 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml +++ b/tripleo_ansible/roles/tripleo_cephadm/files/ceph_spec.yaml @@ -8,6 +8,12 @@ placement: hosts: - standalone.localdomain --- +service_type: mgr +service_name: mgr +placement: + hosts: + - standalone.localdomain +--- service_type: osd service_id: standalone_drive_group placement: diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml index f385233d3..16b076cda 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/apply_spec.yaml @@ -44,6 +44,9 @@ when: - not (ceph_orch_status.stdout | from_json).available +- name: Set container images used by the applied services + include_tasks: config_container_images.yaml + - name: Apply spec command: "{{ tripleo_cephadm_ceph_cli }} orch apply --in-file {{ tripleo_cephadm_container_spec }}" register: tripleo_cephadm_apply_spec_out @@ -53,3 +56,7 @@ debug: msg: "{{ tripleo_cephadm_apply_spec_out }}" when: tripleo_cephadm_verbose + +- name: Wait for the expected number of osds to be running + include_tasks: wait_for_expected_num_osds.yaml + when: tripleo_cephadm_wait_for_osds diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/bootstrap.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/bootstrap.yaml index 9e54d2f0a..f9ca27eb9 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/tasks/bootstrap.yaml +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/bootstrap.yaml @@ -59,9 +59,7 @@ --fsid {{ tripleo_cephadm_fsid }} \ {% if tripleo_cephadm_spec_on_bootstrap %}--apply-spec {{ tripleo_cephadm_spec }} \{% endif %} {% if tripleo_cephadm_bootstrap_conf_stat.stat.exists %}--config {{ tripleo_cephadm_bootstrap_conf }} \{% endif %} - {% if not tripleo_cephadm_dashboard_enabled %} --skip-monitoring-stack --skip-dashboard \ - {% endif %} --mon-ip {{ tripleo_cephadm_first_mon_ip }} register: cephadm_bootstrap become: true diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/config_container_images.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/config_container_images.yaml new file mode 100644 index 000000000..8e7effa3b --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/config_container_images.yaml @@ -0,0 +1,57 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: set default container image in ceph configuration + command: | + {{ tripleo_cephadm_ceph_cli }} config set global container_image \ + {{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} + changed_when: false + become: true + +- name: set container image base in ceph configuration + command: | + {{ tripleo_cephadm_ceph_cli }} config set mgr mgr/cephadm/container_image_base \ + {{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }} + changed_when: false + become: true + +- name: set dashboard container image in ceph mgr configuration + when: tripleo_cephadm_dashboard_enabled | bool + become: true + block: + - name: set alertmanager container image in ceph configuration + command: | + {{ tripleo_cephadm_ceph_cli }} config set mgr mgr/cephadm/container_image_alertmanager \ + {{ tripleo_cephadm_alertmanager_container_image }} + changed_when: false + + - name: set grafana container image in ceph configuration + command: | + {{ tripleo_cephadm_ceph_cli }} config set mgr mgr/cephadm/container_image_grafana \ + {{ tripleo_cephadm_grafana_container_image }} + changed_when: false + + - name: set node-exporter container image in ceph configuration + command: | + {{ tripleo_cephadm_ceph_cli }} config set mgr mgr/cephadm/container_image_node_exporter \ + {{ tripleo_cephadm_node_exporter_container_image }} + changed_when: false + + - name: set prometheus container image in ceph configuration + command: | + {{ tripleo_cephadm_ceph_cli }} config set mgr mgr/cephadm/container_image_prometheus \ + {{ tripleo_cephadm_prometheus_container_image }} + changed_when: false diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/configure_dashboard_backends.yml b/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/configure_dashboard_backends.yml new file mode 100644 index 000000000..534cfe61d --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/configure_dashboard_backends.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Get the current mgr + command: | + {{ container_cli }} ps -a -f 'name=ceph-?(.*)-mgr.*' --format \{\{\.Names\}\} + register: ceph_mgr + become: true + delegate_to: "{{ dashboard_backend }}" + +- name: Check the resulting mgr container instance + debug: + msg: "{{ ceph_mgr.stdout | regex_replace('^ceph-?(.*)-mgr.', '') }}" + when: tripleo_cephadm_verbose + +- name: config the current dashboard backend + command: | + {{ tripleo_cephadm_ceph_cli }} config set \ + mgr mgr/dashboard/{{ ceph_mgr.stdout | regex_replace('^ceph-?(.*)-mgr.', '') }}/server_addr \ + {{ hostvars[dashboard_backend][tripleo_ceph_dashboard_net] }} + become: true + changed_when: false + vars: + tripleo_ceph_dashboard_net: "{{ service_net_map['ceph_dashboard_network'] + '_ip' }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/dashboard.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/dashboard.yaml new file mode 100644 index 000000000..af0e8b70b --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/dashboard.yaml @@ -0,0 +1,108 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Get ceph_cli + include_tasks: ceph_cli.yaml + vars: + mount_spec: true + +- name: Configure the Ceph Dashboard port + become: true + block: + - name: "set the dashboard port ({{ tripleo_cephadm_dashboard_port }})" + command: "{{ tripleo_cephadm_ceph_cli }} config set mgr mgr/dashboard/server_port {{ tripleo_cephadm_dashboard_port }}" + changed_when: false + - name: "set the dashboard SSL port ({{ dashboard_port }})" + command: "{{ tripleo_cephadm_ceph_cli }} config set mgr mgr/dashboard/ssl_server_port {{ tripleo_cephadm_dashboard_port }}" + run_once: true + +- name: disable SSL for dashboard + become: true + command: "{{ tripleo_cephadm_ceph_cli }} config set mgr mgr/dashboard/ssl false" + when: tripleo_cephadm_dashboard_protocol == "http" + +- name: Configure SSL key/cert for the Ceph Dashboard if provided + become: true + block: + - name: enable SSL for dashboard + command: "{{ tripleo_cephadm_ceph_cli }} config set mgr mgr/dashboard/ssl true" + run_once: true + + - name: import dashboard certificate file + command: "{{ tripleo_cephadm_ceph_cli }} config-key set mgr/dashboard/crt -i {{ tripleo_cephadm_dashboard_crt }}" + changed_when: false + + - name: import dashboard certificate key + command: "{{ tripleo_cephadm_ceph_cli }} config-key set mgr/dashboard/key -i {{ tripleo_cephadm_dashboard_key }}" + changed_when: false + when: tripleo_cephadm_dashboard_protocol == "https" and + tripleo_cephadm_dashboard_crt | length > 0 and + tripleo_cephadm_dashboard_key | length > 0 + +- include_tasks: configure_dashboard_backends.yml + with_items: "{{ groups['ceph_mgr'] | default(groups['ceph_mon']) }}" + vars: + dashboard_backend: '{{ item }}' + +# Restart the dashboard and refresh the config (backends and port) +- name: Restart the Ceph dashboard + become: true + block: + - name: disable mgr dashboard module (restart) + command: "{{ tripleo_cephadm_ceph_cli }} mgr module disable dashboard" + - name: enable mgr dashboard module (restart) + command: "{{ tripleo_cephadm_ceph_cli }} mgr module enable dashboard" + +- name: Configure Monitoring Stack + become: true + block: + - name: get Grafana instance(s) addresses + set_fact: + grafana_server_addrs: "{{ (grafana_server_addr | default([])) | union([hostvars[item][tripleo_ceph_dashboard_net]]) }}" + loop: "{{ groups['ceph_mgr'] | list }}" + vars: + tripleo_ceph_dashboard_net: "{{ service_net_map['ceph_dashboard_network'] + '_ip' }}" + - name: set grafana api user + command: "{{ tripleo_cephadm_ceph_cli }} dashboard set-grafana-api-username {{ tripleo_cephadm_grafana_admin_user }}" + - name: set grafana api password + command: "{{ tripleo_cephadm_ceph_cli }} dashboard set-grafana-api-password {{ tripleo_cephadm_grafana_admin_password }}" + - name: disable ssl verification for grafana + command: "{{ tripleo_cephadm_ceph_cli }} dashboard set-grafana-api-ssl-verify False" + changed_when: false + when: + - tripleo_cephadm_dashboard_protocol == "https" + - name: set alertmanager host + command: | + {{ tripleo_cephadm_ceph_cli }} dashboard set-alertmanager-api-host http://{{ grafana_server_addrs | first }}:9093 + - name: set prometheus host + command: | + {{ tripleo_cephadm_ceph_cli }} dashboard set-prometheus-api-host \ + http://{{ grafana_server_addrs | first }}:{{ tripleo_cephadm_prometheus_port }} + - name: config grafana api url vip + command: | + {{ tripleo_cephadm_ceph_cli }} dashboard set-grafana-api-url \ + {{ tripleo_cephadm_dashboard_protocol }}://{{ tripleo_cephadm_dashboard_frontend_vip }}:{{ tripleo_cephadm_grafana_port }} + changed_when: false + when: "{{ tripleo_cephadm_dashboard_frontend_vip is defined and tripleo_cephadm_dashboard_frontend_vip |length > 0 }}" + +- name: Restart the Ceph dashboard + become: true + block: + - name: disable mgr dashboard module (restart) + command: "{{ tripleo_cephadm_ceph_cli }} mgr module disable dashboard" + + - name: enable mgr dashboard module (restart) + command: "{{ tripleo_cephadm_ceph_cli }} mgr module enable dashboard" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/grafana.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/grafana.yaml new file mode 100644 index 000000000..8e3c65671 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/dashboard/grafana.yaml @@ -0,0 +1,55 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# TODO: MKSPEC TO DEPLOY GRAFANA +# - name: Deploy the three, unmanaged grafana instances via the orchestrator +# mkspec: +# service_type: grafana +# service_name: grafana +# placement: +# hosts: +# - ctr1 +# - ctr2 +# - ctr3 +# unmanaged: true + +- name: Get the current mgr addr + set_fact: + grafana_addr: "{{ hostvars[dashboard_backend][tripleo_ceph_dashboard_net] }}" + vars: + tripleo_ceph_dashboard_net: "{{ service_net_map['ceph_dashboard_network'] + '_ip' }}" + delegate_to: "{{ dashboard_backend }}" + +- name: Render config files + block: + - name: Configure grafana + template: + src: grafana.json.j2 + dest: "/tmp/grafana.json" + become: true + delegate_to: "{{ dashboard_backend }}" + +- name: Reconfigure grafana component + shell: | + {{ tripleo_cephadm_bin }} \ + --image {{ tripleo_cephadm_grafana_container_image }} \ + deploy \ + --name grafana.{{ dashboard_backend }} \ + --fsid {{ tripleo_cephadm_fsid }} \ + --config-json /tmp/grafana.json + register: cephadm_grafana + become: true + delegate_to: "{{ dashboard_backend }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml new file mode 100644 index 000000000..842e40ce1 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml @@ -0,0 +1,42 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Using two tasks with a boolean due to the ansible bug: https://github.com/ansible/ansible/issues/68364 +- name: Collect the host and build the resulting host list + set_fact: + _hosts: "{{ _hosts|default([]) + [ hostvars[item].canonical_hostname ] }}" + with_items: "{{ groups['ceph_nfs'] }}" + when: tripleo_cephadm_fqdn | bool + +- name: Collect the host and build the resulting host list + set_fact: + _hosts: "{{ _hosts|default([]) + [ hostvars[item].inventory_hostname ] }}" + with_items: "{{ groups['ceph_nfs'] }}" + when: not tripleo_cephadm_fqdn | bool + +- name: Create the ganesha Daemon spec definition + become: true + ceph_mkspec: + service_type: nfs + apply: true + hosts: "{{ _hosts }}" + render_path: "{{ tripleo_cephadm_spec_home }}" + spec: + namespace: 'ganesha' + pool: "{{ cephfs_data_pool }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/mds.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/mds.yaml new file mode 100644 index 000000000..66edac7ce --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/mds.yaml @@ -0,0 +1,77 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/ +# https://github.com/ceph/ceph-ansible/blob/master/library/ceph_pool.py + +# Using two tasks with a boolean due to the ansible bug: https://github.com/ansible/ansible/issues/68364 +- name: Collect the host and build the resulting host list + set_fact: + _hosts: "{{ _hosts|default([]) + [ hostvars[item].canonical_hostname ] }}" + with_items: "{{ groups['ceph_mds'] }}" + when: tripleo_cephadm_fqdn | bool + +- name: Collect the host and build the resulting host list + set_fact: + _hosts: "{{ _hosts|default([]) + [ hostvars[item].inventory_hostname ] }}" + with_items: "{{ groups['ceph_mds'] }}" + when: not tripleo_cephadm_fqdn | bool + +- name: Create the mds Daemon spec definition + become: true + ceph_mkspec: + service_type: mds + apply: true + hosts: "{{ _hosts }}" + render_path: "{{ tripleo_cephadm_spec_home }}" + register: spc + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + +- name: create filesystem pools + ceph_pool: + name: "{{ item.name }}" + cluster: "{{ tripleo_cephadm_cluster }}" + pg_num: "{{ item.pg_num | default(omit) }}" + pgp_num: "{{ item.pgp_num | default(omit) }}" + size: "{{ item.size | default(omit) }}" + min_size: "{{ item.min_size | default(omit) }}" + pool_type: "{{ item.type | default('replicated') }}" + rule_name: "{{ item.rule_name | default(omit) }}" + erasure_profile: "{{ item.erasure_profile | default(omit) }}" + pg_autoscale_mode: "{{ item.pg_autoscale_mode | default(omit) }}" + target_size_ratio: "{{ item.target_size_ratio | default(omit) }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + with_items: "{{ cephfs_pools }}" + become: true + vars: + cephfs_pools: + - "{{ cephfs_data_pool }}" + - "{{ cephfs_metadata_pool }}" + +- name: create ceph filesystem + ceph_fs: + name: "{{ cephfs|default('cephfs') }}" + cluster: "{{ tripleo_cephadm_cluster }}" + data: "{{ cephfs_data_pool.name }}" + metadata: "{{ cephfs_metadata_pool.name }}" + become: true + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/monitoring.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/monitoring.yaml new file mode 100644 index 000000000..1dc9ff8bf --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/monitoring.yaml @@ -0,0 +1,64 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Build and apply the monitoring stack daemons + block: + # Using two tasks with a boolean due to the ansible bug: https://github.com/ansible/ansible/issues/68364 + - name: Collect the host and build the resulting host list + set_fact: + _hosts: "{{ _hosts|default([]) + [ hostvars[item].canonical_hostname ] }}" + with_items: "{{ groups['ceph_mgr'] }}" + when: tripleo_cephadm_fqdn | bool + + - name: Collect the host and build the resulting host list + set_fact: + _hosts: "{{ _hosts|default([]) + [ hostvars[item].inventory_hostname ] }}" + with_items: "{{ groups['ceph_mgr'] }}" + when: not tripleo_cephadm_fqdn | bool + + - name: Apply node-exporter(s) + become: true + ceph_mkspec: + service_type: "node-exporter" + service_id: "node-exporter" + service_name: "node-exporter" + apply: true + host_pattern: "*" + render_path: "{{ tripleo_cephadm_spec_home }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + + - name: Create the monitoring stack Daemon spec definition + become: true + ceph_mkspec: + service_type: "{{ item }}" + service_id: "{{ item }}" + service_name: "{{ item }}" + apply: true + hosts: "{{ _hosts }}" + render_path: "{{ tripleo_cephadm_spec_home }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + with_items: + - "alertmanager" + - "prometheus" + - "grafana" + when: tripleo_cephadm_dashboard_enabled | bool + +- include_tasks: dashboard/dashboard.yaml + when: tripleo_cephadm_dashboard_enabled | bool diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/post.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/post.yaml new file mode 100644 index 000000000..220f7c436 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/post.yaml @@ -0,0 +1,48 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Get ceph_cli + include_tasks: ceph_cli.yaml + vars: + mount_spec: true + +- name: Get the ceph orchestrator status + command: "{{ tripleo_cephadm_ceph_cli }} orch status --format json" + register: ceph_orch_status + become: true + +- name: Fail if ceph orchestrator is not available + fail: + msg: "'ceph orch status' returned {{ ceph_orch_status.stdout | from_json }}" + when: + - not (ceph_orch_status.stdout | from_json).available + +- name: Show the status of the resulting deployed Ceph cluster + block: + - name: Show the deployed daemons + command: "{{ tripleo_cephadm_ceph_cli }} orch ls --export" + become: true + register: ceph_orch_ls + - name: Deployed daemons + debug: + msg: "{{ ceph_orch_ls.stdout_lines }}" + - name: Print the status of the deployed Ceph cluster + command: "{{ tripleo_cephadm_ceph_cli }} -s" + become: true + register: ceph_health + - name: Ceph cluster status + debug: + msg: "{{ ceph_health.stdout_lines }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/rgw.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/rgw.yaml new file mode 100644 index 000000000..54325ebe7 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/rgw.yaml @@ -0,0 +1,48 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Deploy the RGW daemon + block: + # Using two tasks with a boolean due to the ansible bug: https://github.com/ansible/ansible/issues/68364 + - name: Collect the host and build the resulting host list + set_fact: + _hosts: "{{ _hosts|default([]) + [ hostvars[item].canonical_hostname ] }}" + with_items: "{{ groups['ceph_rgw'] }}" + when: tripleo_cephadm_fqdn | bool + + - name: Collect the host and build the resulting host list + set_fact: + _hosts: "{{ _hosts|default([]) + [ hostvars[item].inventory_hostname ] }}" + with_items: "{{ groups['ceph_rgw'] }}" + when: not tripleo_cephadm_fqdn | bool + + - name: Create the RGW Daemon spec definition + become: true + ceph_mkspec: + service_type: rgw + apply: true + hosts: "{{ _hosts }}" + spec: + rgw_frontend_port: 8080 + rgw_realm: 'default' + rgw_zone: 'default' + render_path: "{{ tripleo_cephadm_spec_home }}" + register: spc + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + when: + - tripleo_enabled_services | intersect(['ceph_rgw']) diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/wait_for_expected_num_osds.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/wait_for_expected_num_osds.yaml new file mode 100644 index 000000000..ebf832c99 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/wait_for_expected_num_osds.yaml @@ -0,0 +1,24 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Wait for expected number of osds to be running + shell: "{{ tripleo_cephadm_ceph_cli }} status --format json | jq .osdmap.num_up_osds" + register: ceph_status + become: true + until: (ceph_status.stdout | int) >= ((tripleo_cephadm_num_osd_expected | int) | default(1)) + retries: "{{ tripleo_cephadm_wait_for_osds_retries }}" + delay: "{{ tripleo_cephadm_wait_for_osds_delay }}" + ignore_errors: "{{ tripleo_cephadm_wait_for_osds_ignore_errors }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/templates/ceph.conf.j2 b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph.conf.j2 index 3f443d3b1..6fd322b62 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/templates/ceph.conf.j2 +++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph.conf.j2 @@ -2,6 +2,13 @@ # {{ ansible_managed }} # Generated by tripleo_cephadm for initial bootstrap of first Ceph Mon +{# MACRO AREA #} +{% macro render_map(root) -%} +{% for key, value in root.items() %} +{{ key}} = {{ value }} +{% endfor %} +{% endmacro %} + [global] fsid = {{ tripleo_cephadm_fsid }} mon host = {{ tripleo_cephadm_first_mon_ip }} @@ -11,3 +18,22 @@ public network = {{ public_network | regex_replace(' ', '') }} {% if cluster_network is defined %} cluster network = {{ cluster_network | regex_replace(' ', '') }} {% endif %} + +{# Build CephConfigOverrides #} +{% for key, value in (ceph_conf_overrides | default({})).items() %} +{% if not (['global', 'mon', 'mgr', 'osd', 'mds', 'client']) | intersect([key]) %} +{{ key }} = {{ value }} +{% elif (['global']) | intersect([key]) %} +{# Merge global defined values into this section #} +{{ render_map(value) }} +{% endif %} +{% endfor %} + +{# Build the remaining sections #} +{% for key, value in (ceph_conf_overrides | default({})).items() %} +{% if (['mon', 'mgr', 'osd', 'mds', 'client']) | intersect([key]) %} +[{{ key }}] +{# Render the current section #} +{{ render_map(value) }} +{% endif %} +{% endfor %} diff --git a/tripleo_ansible/roles/tripleo_cephadm/templates/grafana.json.j2 b/tripleo_ansible/roles/tripleo_cephadm/templates/grafana.json.j2 new file mode 100644 index 000000000..4ab92977a --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/templates/grafana.json.j2 @@ -0,0 +1,39 @@ +{ + "files": { + "grafana.ini": [ + "[users]", + " default_theme = light", + "[auth.anonymous]", + " enabled = true", + " org_name = 'Main Org.'", + " org_role = 'Viewer'", + "[server]", + " protocol = {{ tripleo_cephadm_dashboard_protocol }}", + " cert_file = /etc/grafana/certs/cert_file", + " cert_key = /etc/grafana/certs/cert_key", + " http_port = {{ tripleo_cephadm_grafana_port }}", + " http_addr = {{ grafana_addr }}", + "[security]", + " admin_user = {{ tripleo_cephadm_grafana_admin_user }}", + " admin_password = {{ tripleo_cephadm_grafana_admin_password }}", + " allow_embedding = true" + ], + "provisioning/datasources/ceph-dashboard.yml": [ + "deleteDatasources:", + " - name: 'Dashboard'", + " orgId: 1", + " ", + "datasources:", + " - name: 'Dashboard'", + " type: 'prometheus'", + " access: 'proxy'", + " orgId: 1", + " url: 'http://localhost:{{ tripleo_cephadm_prometheus_port }}'", + " basicAuth: false", + " isDefault: true", + " editable: false" + ], + "certs/cert_file": [], + "certs/cert_key": [] + } +} diff --git a/tripleo_ansible/roles/tripleo_run_cephadm/tasks/prepare.yml b/tripleo_ansible/roles/tripleo_run_cephadm/tasks/prepare.yml index 738441651..486d830d9 100644 --- a/tripleo_ansible/roles/tripleo_run_cephadm/tasks/prepare.yml +++ b/tripleo_ansible/roles/tripleo_run_cephadm/tasks/prepare.yml @@ -108,5 +108,6 @@ tripleo_cephadm_spec_on_bootstrap: false tripleo_cephadm_pools: {{ openstack_pools.get('openstack_pools', []) }} tripleo_cephadm_keys: {{ keys.get('keys',[]) }} - dashboard_frontend_vip: {{ grafana_vip|default() }} + tripleo_cephadm_dashboard_frontend_vip: {{ grafana_vip|default() }} service_net_map: {{ service_net_map|default({}) }} + tripleo_enabled_services: {{ enabled_services | default([]) }} diff --git a/tripleo_ansible/tests/modules/test_ceph_mkspec.py b/tripleo_ansible/tests/modules/test_ceph_mkspec.py new file mode 100644 index 000000000..5762a5b4f --- /dev/null +++ b/tripleo_ansible/tests/modules/test_ceph_mkspec.py @@ -0,0 +1,55 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Test the methods of the ceph_mkspec module""" + + +from tripleo_ansible.ansible_plugins.modules import ceph_mkspec +try: + from ansible.module_utils.ca_common import generate_ceph_cmd +except ImportError: + from tripleo_ansible.ansible_plugins.module_utils.ca_common import generate_ceph_cmd + +try: + from ansible.module_utils import ceph_spec +except ImportError: + from tripleo_ansible.ansible_plugins.module_utils import ceph_spec + +from tripleo_ansible.tests import base as tests_base + + +class TestCephMKSpec(tests_base.TestCase): + ''' + Test the methods of the ceph_spec_bootstrap module + ''' + + def test_generate_orch_cli(self): + ''' + Test the cmd generation run against the ceph cluster when apply: true + is passed to the module. + This command is supposed to use the orchestrator and apply the spec + rendered in a given input_path + ''' + + input_path = "/tmp/specfile" + cluster = "ceph" + container_image = "quay.ceph.io/ceph-ci/daemon:latest" + args = ['apply', '--in-file', input_path] + + expected_cli_cmd = generate_ceph_cmd(sub_cmd=['orch'], args=args, + spec_path=input_path, cluster=cluster, + container_image=container_image) + + gen_cli_cmd = ceph_mkspec.generate_orch_cli(cluster, input_path, container_image) + self.assertEqual(expected_cli_cmd, gen_cli_cmd)