From 0c40adf136aea7bdcfd22e79337568157b51ad9c Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Tue, 16 Mar 2021 12:15:07 +0100 Subject: [PATCH] Configure crush rules according to the detected device classes This change introduces tasks, modules and the proper flow needed to manage crush rules (when specified via tht). The implemented flow reflects the steps done by ceph-ansible when crush rules are configured. The modules are imported from ceph-ansible and a just a few changes are added to make them cephadm/spec compatible. The ceph_cli is extended in order to allow the crush_rule tasks to run commands using --admin-daemon and the mon socket. Change-Id: If51cdfab825c86cd78da4520cb130d93148a1d43 --- .ansible-lint | 1 + .../modules/ceph_crush_rule.py | 242 ++++++++++++++++++ tripleo_ansible/playbooks/cephadm.yml | 6 + .../roles/tripleo_cephadm/defaults/main.yml | 1 + .../roles/tripleo_cephadm/tasks/ceph_cli.yaml | 15 +- .../tripleo_cephadm/tasks/crush_rules.yaml | 69 +++++ 6 files changed, 330 insertions(+), 4 deletions(-) create mode 100644 tripleo_ansible/ansible_plugins/modules/ceph_crush_rule.py create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/crush_rules.yaml diff --git a/.ansible-lint b/.ansible-lint index 143e8aa30..4a6fc9f99 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -14,6 +14,7 @@ mock_roles: mock_modules: - baremetal_nodes_validate - baremetal_register_or_update_nodes + - ceph_crush_rule - ceph_dashboard_user - ceph_key - ceph_fs diff --git a/tripleo_ansible/ansible_plugins/modules/ceph_crush_rule.py b/tripleo_ansible/ansible_plugins/modules/ceph_crush_rule.py new file mode 100644 index 000000000..825c3d17a --- /dev/null +++ b/tripleo_ansible/ansible_plugins/modules/ceph_crush_rule.py @@ -0,0 +1,242 @@ +# Copyright 2020, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Included from: https://github.com/ceph/ceph-ansible/blob/master/library/ceph_crush_rule.py + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +from ansible.module_utils.basic import AnsibleModule +try: + from ansible.module_utils.ca_common import exit_module, generate_ceph_cmd, is_containerized, exec_command +except ImportError: + from module_utils.ca_common import exit_module, generate_ceph_cmd, is_containerized, exec_command +import datetime +import json +import yaml + + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = """ +--- +module: ceph_crush_rule +short_description: Manage Ceph Crush Replicated/Erasure Rule +version_added: "2.8" +description: + - Manage Ceph Crush rule(s) creation, deletion and updates. +options: + name: + description: + - name of the Ceph Crush rule. + required: true + type: str + cluster: + description: + - The ceph cluster name. + required: false + default: ceph + type: str + state: + description: + If 'present' is used, the module creates a rule if it doesn't + exist or update it if it already exists. + If 'absent' is used, the module will simply delete the rule. + If 'info' is used, the module will return all details about the + existing rule (json formatted). + required: false + choices: ['present', 'absent', 'info'] + default: present + type: str + rule_type: + description: + - The ceph CRUSH rule type. + required: false + choices: ['replicated', 'erasure'] + required: false + type: str + bucket_root: + description: + - The ceph bucket root for replicated rule. + required: false + type: str + bucket_type: + description: + - The ceph bucket type for replicated rule. + required: false + choices: ['osd', 'host', 'chassis', 'rack', 'row', 'pdu', 'pod', 'room', 'datacenter', 'zone', 'region', 'root'] + type: str + device_class: + description: + - The ceph device class for replicated rule. + required: false + type: str + profile: + description: + - The ceph erasure profile for erasure rule. + required: false + type: str +author: + - Dimitri Savineau +""" + +EXAMPLES = ''' +- name: create a Ceph Crush replicated rule + ceph_crush_rule: + name: foo + bucket_root: default + bucket_type: host + device_class: ssd + rule_type: replicated + +- name: create a Ceph Crush erasure rule + ceph_crush_rule: + name: foo + profile: bar + rule_type: erasure + +- name: get a Ceph Crush rule information + ceph_crush_rule: + name: foo + state: info + +- name: delete a Ceph Crush rule + ceph_crush_rule: + name: foo + state: absent +''' + +RETURN = '''# ''' + + +def create_rule(module, container_image=None): + ''' + Create a new crush replicated/erasure rule + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + rule_type = module.params.get('rule_type') + bucket_root = module.params.get('bucket_root') + bucket_type = module.params.get('bucket_type') + device_class = module.params.get('device_class') + profile = module.params.get('profile') + + if rule_type == 'replicated': + args = ['create-replicated', name, bucket_root, bucket_type] + if device_class: + args.append(device_class) + else: + args = ['create-erasure', name] + if profile: + args.append(profile) + + cmd = generate_ceph_cmd(['osd', 'crush', 'rule'], args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def get_rule(module, container_image=None): + ''' + Get existing crush rule + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['dump', name, '--format=json'] + + cmd = generate_ceph_cmd(['osd', 'crush', 'rule'], args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def remove_rule(module, container_image=None): + ''' + Remove a crush rule + ''' + + cluster = module.params.get('cluster') + name = module.params.get('name') + + args = ['rm', name] + + cmd = generate_ceph_cmd(['osd', 'crush', 'rule'], args, spec_path=None, cluster=cluster, container_image=container_image) + + return cmd + + +def main(): + module = AnsibleModule( + argument_spec=yaml.safe_load(DOCUMENTATION)['options'], + supports_check_mode=True, + required_if=[ + ('state', 'present', ['rule_type']), + ('rule_type', 'replicated', ['bucket_root', 'bucket_type']), + ('rule_type', 'erasure', ['profile']) + ] + ) + + # Gather module parameters in variables + name = module.params.get('name') + state = module.params.get('state') + rule_type = module.params.get('rule_type') + + if module.check_mode: + module.exit_json( + changed=False, + stdout='', + stderr='', + rc=0, + start='', + end='', + delta='', + ) + + startd = datetime.datetime.now() + changed = False + + # will return either the image name or None + container_image = is_containerized() + + if state == "present": + rc, cmd, out, err = exec_command(module, get_rule(module, container_image=container_image)) + if rc != 0: + rc, cmd, out, err = exec_command(module, create_rule(module, container_image=container_image)) + changed = True + else: + rule = json.loads(out) + if (rule['type'] == 1 and rule_type == 'erasure') or (rule['type'] == 3 and rule_type == 'replicated'): + module.fail_json(msg="Can not convert crush rule {} to {}".format(name, rule_type), changed=False, rc=1) + + elif state == "absent": + rc, cmd, out, err = exec_command(module, get_rule(module, container_image=container_image)) + if rc == 0: + rc, cmd, out, err = exec_command(module, remove_rule(module, container_image=container_image)) + changed = True + else: + rc = 0 + out = "Crush Rule {} doesn't exist".format(name) + + elif state == "info": + rc, cmd, out, err = exec_command(module, get_rule(module, container_image=container_image)) + + exit_module(module=module, out=out, rc=rc, cmd=cmd, err=err, startd=startd, changed=changed) + + +if __name__ == '__main__': + main() diff --git a/tripleo_ansible/playbooks/cephadm.yml b/tripleo_ansible/playbooks/cephadm.yml index 5859f2548..94d4e8bbd 100644 --- a/tripleo_ansible/playbooks/cephadm.yml +++ b/tripleo_ansible/playbooks/cephadm.yml @@ -32,6 +32,12 @@ tasks_from: apply_spec when: not tripleo_cephadm_spec_on_bootstrap + - name: Set crush rules if provided + import_role: + name: tripleo_cephadm + tasks_from: crush_rules + when: tripleo_cephadm_crush_rules | length > 0 + - name: Create Pools import_role: name: tripleo_cephadm diff --git a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml index cf1eb4d8f..b3fa9ffb0 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml +++ b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml @@ -39,5 +39,6 @@ tripleo_cephadm_predeployed: true tripleo_cephadm_conf_overrides: {} tripleo_cephadm_fsid_list: [] tripleo_cephadm_fqdn: false +tripleo_cephadm_crush_rules: [] # todo(fultonj) add is_hci boolean for target memory # https://lists.ceph.io/hyperkitty/list/dev@ceph.io/thread/Z77XO23JPXDNHKM7IG6UN4URYKA6L7VH/ diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml index b3305f652..af4efce19 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml @@ -19,9 +19,16 @@ tripleo_cephadm_ceph_cli: >- {{ tripleo_cephadm_container_cli }} run --rm {{ tripleo_cephadm_container_options }} --volume {{ tripleo_cephadm_config_home }}:/etc/ceph:z - {% if mount_spec|default(false) %} --volume {{ tripleo_cephadm_spec }}:{{ tripleo_cephadm_container_spec }}:z {% endif %} + {% if mount_spec|default(false) %} + --volume {{ tripleo_cephadm_spec }}:{{ tripleo_cephadm_container_spec }}:z + {% endif %} + {% if admin_daemon|default(false) %} + --volume /var/run/ceph/{{ tripleo_cephadm_fsid }}:/var/run/ceph:z + {% endif %} --entrypoint {{ ceph_command | default('ceph') }} {{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} - {% if ceph_command|default('ceph') == 'ceph' %} - --fsid {{ tripleo_cephadm_fsid }} -c {{ tripleo_cephadm_conf }} -k {{ tripleo_cephadm_admin_keyring }} - {% endif %} + {% if ceph_command|default('ceph') == 'ceph' -%} + {% if not admin_daemon|default(false) -%} + --fsid {{ tripleo_cephadm_fsid }} -c {{ tripleo_cephadm_conf }} -k {{ tripleo_cephadm_admin_keyring }} + {%- endif %} + {%- endif %} diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/crush_rules.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/crush_rules.yaml new file mode 100644 index 000000000..819dc95e7 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/crush_rules.yaml @@ -0,0 +1,69 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/ +# https://github.com/ceph/ceph-ansible/blob/master/library/ceph_pool.py + +- name: Get ceph_cli + include_tasks: ceph_cli.yaml + vars: + admin_daemon: true + +- name: create configured crush rules + ceph_crush_rule: + name: "{{ item.name }}" + cluster: "{{ tripleo_cephadm_cluster }}" + rule_type: replicated + bucket_root: "{{ item.root }}" + bucket_type: "{{ item.type }}" + device_class: "{{ item.class | default(omit) }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + with_items: "{{ tripleo_cephadm_crush_rules | unique }}" + run_once: true + +- name: get id for new default crush rule + ceph_crush_rule: + name: "{{ item.name }}" + cluster: "{{ tripleo_cephadm_cluster }}" + state: info + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + register: info_ceph_default_crush_rule + with_items: "{{ tripleo_cephadm_crush_rules | unique }}" + run_once: true + when: item.default | default(False) | bool + +- name: set_fact info_ceph_default_crush_rule_yaml, ceph_osd_pool_default_crush_rule_name + set_fact: + info_ceph_default_crush_rule_yaml: "{{ item.stdout | from_json() }}" + ceph_osd_pool_default_crush_rule_name: "{{ (item.stdout | from_json).rule_name }}" + with_items: "{{ info_ceph_default_crush_rule.results }}" + run_once: true + when: not item.get('skipped', false) + +- name: insert new default crush rule into daemon to prevent restart + command: | + {{ tripleo_cephadm_ceph_cli }} --admin-daemon /var/run/ceph/{{ tripleo_cephadm_cluster }}-mon.{{ hostvars[item].canonical_hostname }}.asok \ + config set osd_pool_default_crush_rule {{ info_ceph_default_crush_rule_yaml.rule_id }} + changed_when: false + delegate_to: "{{ item }}" + with_items: "{{ groups['ceph_mon'] }}" + run_once: true + when: + - info_ceph_default_crush_rule_yaml | default('') | length > 0