Merge "Configure crush rules according to the detected device classes"

.ansible-lint

@@ -14,6 +14,7 @@ mock_roles:
 mock_modules:
     - baremetal_nodes_validate
     - baremetal_register_or_update_nodes
+    - ceph_crush_rule
     - ceph_dashboard_user
     - ceph_key
     - ceph_fs

tripleo_ansible/ansible_plugins/modules/ceph_crush_rule.py

@@ -0,0 +1,242 @@
+# Copyright 2020, Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Included from: https://github.com/ceph/ceph-ansible/blob/master/library/ceph_crush_rule.py
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from ansible.module_utils.basic import AnsibleModule
+try:
+    from ansible.module_utils.ca_common import exit_module, generate_ceph_cmd, is_containerized, exec_command
+except ImportError:
+    from module_utils.ca_common import exit_module, generate_ceph_cmd, is_containerized, exec_command
+import datetime
+import json
+import yaml
+
+
+ANSIBLE_METADATA = {
+    'metadata_version': '1.1',
+    'status': ['preview'],
+    'supported_by': 'community'
+}
+
+DOCUMENTATION = """
+---
+module: ceph_crush_rule
+short_description: Manage Ceph Crush Replicated/Erasure Rule
+version_added: "2.8"
+description:
+  - Manage Ceph Crush rule(s) creation, deletion and updates.
+options:
+  name:
+    description:
+      - name of the Ceph Crush rule.
+    required: true
+    type: str
+  cluster:
+    description:
+      - The ceph cluster name.
+    required: false
+    default: ceph
+    type: str
+  state:
+    description:
+      If 'present' is used, the module creates a rule if it doesn't
+      exist or update it if it already exists.
+      If 'absent' is used, the module will simply delete the rule.
+      If 'info' is used, the module will return all details about the
+      existing rule (json formatted).
+    required: false
+    choices: ['present', 'absent', 'info']
+    default: present
+    type: str
+  rule_type:
+    description:
+      - The ceph CRUSH rule type.
+    required: false
+    choices: ['replicated', 'erasure']
+    required: false
+    type: str
+  bucket_root:
+    description:
+      - The ceph bucket root for replicated rule.
+    required: false
+    type: str
+  bucket_type:
+    description:
+      - The ceph bucket type for replicated rule.
+    required: false
+    choices: ['osd', 'host', 'chassis', 'rack', 'row', 'pdu', 'pod', 'room', 'datacenter', 'zone', 'region', 'root']
+    type: str
+  device_class:
+    description:
+      - The ceph device class for replicated rule.
+    required: false
+    type: str
+  profile:
+    description:
+      - The ceph erasure profile for erasure rule.
+    required: false
+    type: str
+author:
+  - Dimitri Savineau <dsavinea@redhat.com>
+"""
+
+EXAMPLES = '''
+- name: create a Ceph Crush replicated rule
+  ceph_crush_rule:
+    name: foo
+    bucket_root: default
+    bucket_type: host
+    device_class: ssd
+    rule_type: replicated
+
+- name: create a Ceph Crush erasure rule
+  ceph_crush_rule:
+    name: foo
+    profile: bar
+    rule_type: erasure
+
+- name: get a Ceph Crush rule information
+  ceph_crush_rule:
+    name: foo
+    state: info
+
+- name: delete a Ceph Crush rule
+  ceph_crush_rule:
+    name: foo
+    state: absent
+'''
+
+RETURN = '''#  '''
+
+
+def create_rule(module, container_image=None):
+    '''
+    Create a new crush replicated/erasure rule
+    '''
+
+    cluster = module.params.get('cluster')
+    name = module.params.get('name')
+    rule_type = module.params.get('rule_type')
+    bucket_root = module.params.get('bucket_root')
+    bucket_type = module.params.get('bucket_type')
+    device_class = module.params.get('device_class')
+    profile = module.params.get('profile')
+
+    if rule_type == 'replicated':
+        args = ['create-replicated', name, bucket_root, bucket_type]
+        if device_class:
+            args.append(device_class)
+    else:
+        args = ['create-erasure', name]
+        if profile:
+            args.append(profile)
+
+    cmd = generate_ceph_cmd(['osd', 'crush', 'rule'], args, spec_path=None, cluster=cluster, container_image=container_image)
+
+    return cmd
+
+
+def get_rule(module, container_image=None):
+    '''
+    Get existing crush rule
+    '''
+
+    cluster = module.params.get('cluster')
+    name = module.params.get('name')
+
+    args = ['dump', name, '--format=json']
+
+    cmd = generate_ceph_cmd(['osd', 'crush', 'rule'], args, spec_path=None, cluster=cluster, container_image=container_image)
+
+    return cmd
+
+
+def remove_rule(module, container_image=None):
+    '''
+    Remove a crush rule
+    '''
+
+    cluster = module.params.get('cluster')
+    name = module.params.get('name')
+
+    args = ['rm', name]
+
+    cmd = generate_ceph_cmd(['osd', 'crush', 'rule'], args, spec_path=None, cluster=cluster, container_image=container_image)
+
+    return cmd
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=yaml.safe_load(DOCUMENTATION)['options'],
+        supports_check_mode=True,
+        required_if=[
+            ('state', 'present', ['rule_type']),
+            ('rule_type', 'replicated', ['bucket_root', 'bucket_type']),
+            ('rule_type', 'erasure', ['profile'])
+        ]
+    )
+
+    # Gather module parameters in variables
+    name = module.params.get('name')
+    state = module.params.get('state')
+    rule_type = module.params.get('rule_type')
+
+    if module.check_mode:
+        module.exit_json(
+            changed=False,
+            stdout='',
+            stderr='',
+            rc=0,
+            start='',
+            end='',
+            delta='',
+        )
+
+    startd = datetime.datetime.now()
+    changed = False
+
+    # will return either the image name or None
+    container_image = is_containerized()
+
+    if state == "present":
+        rc, cmd, out, err = exec_command(module, get_rule(module, container_image=container_image))
+        if rc != 0:
+            rc, cmd, out, err = exec_command(module, create_rule(module, container_image=container_image))
+            changed = True
+        else:
+            rule = json.loads(out)
+            if (rule['type'] == 1 and rule_type == 'erasure') or (rule['type'] == 3 and rule_type == 'replicated'):
+                module.fail_json(msg="Can not convert crush rule {} to {}".format(name, rule_type), changed=False, rc=1)
+
+    elif state == "absent":
+        rc, cmd, out, err = exec_command(module, get_rule(module, container_image=container_image))
+        if rc == 0:
+            rc, cmd, out, err = exec_command(module, remove_rule(module, container_image=container_image))
+            changed = True
+        else:
+            rc = 0
+            out = "Crush Rule {} doesn't exist".format(name)
+
+    elif state == "info":
+        rc, cmd, out, err = exec_command(module, get_rule(module, container_image=container_image))
+
+    exit_module(module=module, out=out, rc=rc, cmd=cmd, err=err, startd=startd, changed=changed)
+
+
+if __name__ == '__main__':
+    main()

tripleo_ansible/playbooks/cephadm.yml

@@ -32,6 +32,12 @@
         tasks_from: apply_spec
       when: not tripleo_cephadm_spec_on_bootstrap
 
+    - name: Set crush rules if provided
+      import_role:
+        name: tripleo_cephadm
+        tasks_from: crush_rules
+      when: tripleo_cephadm_crush_rules | length > 0
+
     - name: Create Pools
       import_role:
         name: tripleo_cephadm

tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml

@@ -43,5 +43,6 @@ tripleo_cephadm_predeployed: true
 tripleo_cephadm_conf_overrides: {}
 tripleo_cephadm_fsid_list: []
 tripleo_cephadm_fqdn: false
+tripleo_cephadm_crush_rules: []
 # todo(fultonj) add is_hci boolean for target memory
 # https://lists.ceph.io/hyperkitty/list/dev@ceph.io/thread/Z77XO23JPXDNHKM7IG6UN4URYKA6L7VH/

tripleo_ansible/roles/tripleo_cephadm/tasks/ceph_cli.yaml

@@ -19,9 +19,16 @@
     tripleo_cephadm_ceph_cli: >-
       {{ tripleo_cephadm_container_cli }} run --rm {{ tripleo_cephadm_container_options }}
       --volume {{ tripleo_cephadm_config_home }}:/etc/ceph:z
-      {% if mount_spec|default(false) %} --volume {{ tripleo_cephadm_spec }}:{{ tripleo_cephadm_container_spec }}:z {% endif %}
+      {% if mount_spec|default(false) %}
+      --volume {{ tripleo_cephadm_spec }}:{{ tripleo_cephadm_container_spec }}:z
+      {% endif %}
+      {% if admin_daemon|default(false) %}
+      --volume /var/run/ceph/{{ tripleo_cephadm_fsid }}:/var/run/ceph:z
+      {% endif %}
       --entrypoint {{ ceph_command | default('ceph') }}
       {{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }}
-      {% if ceph_command|default('ceph') == 'ceph' %}
-      --fsid {{ tripleo_cephadm_fsid }} -c {{ tripleo_cephadm_conf }} -k {{ tripleo_cephadm_admin_keyring }}
-      {% endif %}
+      {% if ceph_command|default('ceph') == 'ceph' -%}
+        {% if not admin_daemon|default(false) -%}
+        --fsid {{ tripleo_cephadm_fsid }} -c {{ tripleo_cephadm_conf }} -k {{ tripleo_cephadm_admin_keyring }}
+        {%- endif %}
+      {%- endif %}

tripleo_ansible/roles/tripleo_cephadm/tasks/crush_rules.yaml

@@ -0,0 +1,69 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Assumes the following module is in ANSIBLE_LIBRARY=/usr/share/ansible/library/
+#   https://github.com/ceph/ceph-ansible/blob/master/library/ceph_pool.py
+
+- name: Get ceph_cli
+  include_tasks: ceph_cli.yaml
+  vars:
+    admin_daemon: true
+
+- name: create configured crush rules
+  ceph_crush_rule:
+    name: "{{ item.name }}"
+    cluster: "{{ tripleo_cephadm_cluster }}"
+    rule_type: replicated
+    bucket_root: "{{ item.root }}"
+    bucket_type: "{{ item.type }}"
+    device_class: "{{ item.class | default(omit) }}"
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
+    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+  with_items: "{{ tripleo_cephadm_crush_rules | unique }}"
+  run_once: true
+
+- name: get id for new default crush rule
+  ceph_crush_rule:
+    name: "{{ item.name }}"
+    cluster: "{{ tripleo_cephadm_cluster }}"
+    state: info
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
+    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+  register: info_ceph_default_crush_rule
+  with_items: "{{ tripleo_cephadm_crush_rules | unique }}"
+  run_once: true
+  when: item.default | default(False) | bool
+
+- name: set_fact info_ceph_default_crush_rule_yaml, ceph_osd_pool_default_crush_rule_name
+  set_fact:
+    info_ceph_default_crush_rule_yaml: "{{ item.stdout | from_json() }}"
+    ceph_osd_pool_default_crush_rule_name: "{{ (item.stdout | from_json).rule_name }}"
+  with_items: "{{ info_ceph_default_crush_rule.results }}"
+  run_once: true
+  when: not item.get('skipped', false)
+
+- name: insert new default crush rule into daemon to prevent restart
+  command: |
+    {{ tripleo_cephadm_ceph_cli }} --admin-daemon /var/run/ceph/{{ tripleo_cephadm_cluster }}-mon.{{ hostvars[item].canonical_hostname }}.asok \
+    config set osd_pool_default_crush_rule {{ info_ceph_default_crush_rule_yaml.rule_id }}
+  changed_when: false
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups['ceph_mon'] }}"
+  run_once: true
+  when:
+    - info_ceph_default_crush_rule_yaml | default('') | length > 0