From 6415174661f8690de738aceff4ad77c9d824cb6f Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Thu, 8 Apr 2021 11:01:31 +0200 Subject: [PATCH] Create ganesha-export-index before the ceph-nfs daemon is deployed When ceph_nfs_rados_backend is provided, as per doc [1], for a fresh setup, we need to make sure to create the Ganesha export index object as an empty object before starting the Ganesha server. This change provides the two tasks, already present in ceph-ansible, to create the index object. In addition, since the ganesha systemd units are managed by pacemaker, this change renders the related j2 templates and prepares the ganesha workdir in the ceph-nfs target hosts. [1] https://docs.openstack.org/manila/latest/contributor/ganesha.html [2] https://github.com/ceph/ceph-ansible/blob/master/roles/ceph-nfs/tasks/start_nfs.yml#L15-L31 Change-Id: Ie89c35238d1bb82a28d560d7f0f718f7ee38044f --- tripleo_ansible/playbooks/cephadm.yml | 4 +- .../roles/tripleo_cephadm/defaults/main.yml | 2 + .../roles/tripleo_cephadm/tasks/ganesha.yaml | 42 --------- .../tasks/ganesha/distribute_keys.yaml | 34 +++++++ .../tasks/ganesha/start_nfs.yaml | 94 +++++++++++++++++++ .../roles/tripleo_cephadm/tasks/nfs.yaml | 78 +++++++++++++++ .../templates/ceph-nfs.service.j2 | 44 +++++++++ .../tripleo_cephadm/templates/ganesha.conf.j2 | 52 ++++++++++ 8 files changed, 306 insertions(+), 44 deletions(-) delete mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml create mode 100644 tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2 create mode 100644 tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2 diff --git a/tripleo_ansible/playbooks/cephadm.yml b/tripleo_ansible/playbooks/cephadm.yml index 94d4e8bbd..c2100b54a 100644 --- a/tripleo_ansible/playbooks/cephadm.yml +++ b/tripleo_ansible/playbooks/cephadm.yml @@ -56,9 +56,9 @@ tasks_from: mds - name: Config Ganesha - import_role: + include_role: name: tripleo_cephadm - tasks_from: ganesha + tasks_from: nfs when: - cephfs_data_pool is defined - cephfs_metadata_pool is defined diff --git a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml index 9cbeda469..1f112e985 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml +++ b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml @@ -45,5 +45,7 @@ tripleo_cephadm_fsid_list: [] tripleo_cephadm_fqdn: false tripleo_cephadm_crush_rules: [] tripleo_cephadm_internal_tls_enabled: false +tripleo_cephadm_nfs_rados_export_index: 'ganesha-export-index' +tripleo_cephadm_ceph_nfs_rados_backend: true # todo(fultonj) add is_hci boolean for target memory # https://lists.ceph.io/hyperkitty/list/dev@ceph.io/thread/Z77XO23JPXDNHKM7IG6UN4URYKA6L7VH/ diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml deleted file mode 100644 index 842e40ce1..000000000 --- a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -# Copyright 2021 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Using two tasks with a boolean due to the ansible bug: https://github.com/ansible/ansible/issues/68364 -- name: Collect the host and build the resulting host list - set_fact: - _hosts: "{{ _hosts|default([]) + [ hostvars[item].canonical_hostname ] }}" - with_items: "{{ groups['ceph_nfs'] }}" - when: tripleo_cephadm_fqdn | bool - -- name: Collect the host and build the resulting host list - set_fact: - _hosts: "{{ _hosts|default([]) + [ hostvars[item].inventory_hostname ] }}" - with_items: "{{ groups['ceph_nfs'] }}" - when: not tripleo_cephadm_fqdn | bool - -- name: Create the ganesha Daemon spec definition - become: true - ceph_mkspec: - service_type: nfs - apply: true - hosts: "{{ _hosts }}" - render_path: "{{ tripleo_cephadm_spec_home }}" - spec: - namespace: 'ganesha' - pool: "{{ cephfs_data_pool }}" - environment: - CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" - CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml new file mode 100644 index 000000000..b83c995c8 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml @@ -0,0 +1,34 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: create keyring directory + file: + name: /var/lib/ceph/bootstrap-rgw/ + state: directory + owner: "167" + group: "167" + mode: "0755" + delegate_to: "{{ nfs_node }}" + +- name: copy ceph key(s) if needed + copy: + dest: "{{ item.item.path }}" + content: "{{ item.stdout + '\n' }}" + owner: "167" + group: "167" + mode: "0755" + with_items: "{{ _rgw_keys.results }}" + delegate_to: "{{ nfs_node }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml new file mode 100644 index 000000000..af40e4fb8 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml @@ -0,0 +1,94 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: dbus related tasks + block: + - name: create dbus service file + copy: + content: "{{ dbus_ganesha_file.stdout }}" + dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf + owner: "root" + group: "root" + mode: "0644" + + - name: reload dbus configuration + command: "killall -SIGHUP dbus-daemon" + when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool + delegate_to: "{{ nfs_node }}" + +- name: Render config files + delegate_to: "{{ nfs_node }}" + block: + - name: Make sure /etc/ganesha exists + file: + path: /etc/ganesha + state: directory + owner: root + group: root + mode: "0755" + + - name: create exports directory + file: + path: /etc/ganesha/export.d + state: directory + owner: "root" + group: "root" + mode: "0755" + when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool + + - name: create exports dir index file + copy: + content: "" + force: false + dest: /etc/ganesha/export.d/INDEX.conf + owner: "root" + group: "root" + mode: "0644" + when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool + + - name: Render ganesha config file + template: + src: ganesha.conf.j2 + dest: /etc/ganesha/ganesha.conf + become: true + + - name: Render ganesha systemd unit + template: + src: ceph-nfs.service.j2 + dest: /etc/systemd/system/ceph-nfs@.service + owner: "root" + group: "root" + mode: "0644" + become: true + + - name: systemd start nfs container + systemd: + name: ceph-nfs@{{ tripleo_cephadm_ceph_nfs_service_suffix | default(ansible_facts['hostname']) }} + state: started + enabled: yes + masked: no + daemon_reload: yes + when: + - tripleo_cephadm_ceph_nfs_enable_service | bool + + - name: start nfs gateway service + systemd: + name: nfs-ganesha + state: started + enabled: yes + masked: no + when: + - tripleo_cephadm_ceph_nfs_enable_service | bool diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml new file mode 100644 index 000000000..9ed1344ef --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml @@ -0,0 +1,78 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Get ceph_cli + include_tasks: ceph_cli.yaml + vars: + ceph_command: 'rados' + +- name: check if rados index object exists + shell: | + {{ tripleo_cephadm_ceph_cli }} -p {{ cephfs_data_pool.name }} --cluster {{ tripleo_cephadm_cluster }} \ + ls | grep {{ tripleo_cephadm_nfs_rados_export_index }} + changed_when: false + failed_when: false + register: rados_index_exists + check_mode: false + when: tripleo_cephadm_ceph_nfs_rados_backend | bool + +- name: create an empty rados index object + command: | + {{ tripleo_cephadm_ceph_cli }} -p {{ cephfs_data_pool.name }} --cluster {{ tripleo_cephadm_cluster }} \ + put {{ tripleo_cephadm_nfs_rados_export_index }} /dev/null + when: + - tripleo_cephadm_ceph_nfs_rados_backend | bool + - rados_index_exists.rc != 0 + +- name: Extract key + ceph_key: + name: "{{ item.name }}" + output_format: plain + state: info + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + register: _rgw_keys + become: true + with_items: + - { name: "client.bootstrap-rgw", path: "/var/lib/ceph/bootstrap-rgw/{{ tripleo_cephadm_cluster }}.keyring", copy_key: true } + +- name: Distribute Ceph keys to the nfs nodes + include_tasks: ganesha/distribute_keys.yaml + vars: + nfs_node: "{{ node }}" + rgw_keys: "{{ _rgw_keys }}" + loop: "{{ groups['ceph_nfs'] }}" + loop_control: + loop_var: node + +- name: get dbus-1 file + command: | + {{ container_cli }} run --rm --entrypoint=cat \ + {{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }} \ + /etc/dbus-1/system.d/org.ganesha.nfsd.conf + register: _dbus_ganesha + run_once: true + changed_when: false + +- name: Render Ganesha templates and files + include_tasks: ganesha/start_nfs.yaml + vars: + nfs_node: "{{ node }}" + dbus_ganesha_file: "{{ _dbus_ganesha }}" + loop: "{{ groups['ceph_nfs'] }}" + loop_control: + loop_var: node diff --git a/tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2 b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2 new file mode 100644 index 000000000..6414cc17e --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2 @@ -0,0 +1,44 @@ +[Unit] +Description=NFS-Ganesha file server +Documentation=http://github.com/nfs-ganesha/nfs-ganesha/wiki +After=network.target + +[Service] +EnvironmentFile=-/etc/environment +{% if container_cli == 'podman' %} +ExecStartPre=-/usr/bin/rm -f /%t/%n-pid /%t/%n-cid +ExecStartPre=-/usr/bin/{{ container_cli }} rm --storage ceph-nfs-%i +ExecStartPre=-/usr/bin/mkdir -p /var/log/ceph /var/log/ganesha +{% endif %} +ExecStartPre=-/usr/bin/{{ container_cli }} rm ceph-nfs-%i +ExecStartPre={{ '/bin/mkdir' if ansible_facts['os_family'] == 'Debian' else '/usr/bin/mkdir' }} -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha /var/log/ganesha +ExecStart=/usr/bin/{{ container_cli }} run --rm --net=host \ + -d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \ + -v /var/lib/ceph:/var/lib/ceph:z \ + -v /etc/ceph:/etc/ceph:z \ + -v /var/lib/nfs/ganesha:/var/lib/nfs/ganesha:z \ + -v /etc/ganesha:/etc/ganesha:z \ + -v /var/run/ceph:/var/run/ceph:z \ + -v /var/log/ceph:/var/log/ceph:z \ + -v /var/log/ganesha:/var/log/ganesha:z \ + {% if tripleo_cephadm_ceph_nfs_dynamic_exports | bool %} + --privileged \ + -v /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket \ + {% endif -%} + -v /etc/localtime:/etc/localtime:ro \ + -e CLUSTER={{ tripleo_cephadm_cluster }} \ + -e CEPH_DAEMON=NFS \ + -e CONTAINER_IMAGE={{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} \ + --name=ceph-nfs-{{ tripleo_cephadm_ceph_nfs_service_suffix | default(ansible_facts['hostname']) }} \ + {{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} +ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_cli }} rm -f `cat /%t/%n-cid`" +KillMode=none +Restart=always +RestartSec=10s +TimeoutStartSec=120 +TimeoutStopSec=15 +Type=forking +PIDFile=/%t/%n-pid + +[Install] +WantedBy=multi-user.target diff --git a/tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2 b/tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2 new file mode 100644 index 000000000..54fce8853 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2 @@ -0,0 +1,52 @@ +#jinja2: trim_blocks: "true", lstrip_blocks: "true" +# {{ ansible_managed }} + +{% if tripleo_cephadm_ceph_nfs_dynamic_exports | bool and not tripleo_cephadm_ceph_nfs_rados_backend | bool %} +%include /etc/ganesha/export.d/INDEX.conf +{% endif %} + +NFS_Core_Param +{ +{% if tripleo_cephadm_ceph_nfs_bind_addr is defined %} + Bind_Addr={{ tripleo_cephadm_ceph_nfs_bind_addr }}; +{% endif %} +} + +{% if tripleo_cephadm_ceph_nfs_disable_caching | bool or nfs_file_gw | bool %} +EXPORT_DEFAULTS { + Attr_Expiration_Time = 0; +} + +CACHEINODE { + Dir_Chunk = 0; + + NParts = 1; + Cache_Size = 1; +} +{% endif %} + +{% if tripleo_cephadm_ceph_nfs_rados_backend | bool %} +RADOS_URLS { + ceph_conf = '/etc/ceph/{{ tripleo_cephadm_cluster }}.conf'; + userid = "{{ tripleo_cephadm_ceph_nfs_ceph_user }}"; +} +%url rados://{{ cephfs_data_pool.name }}/{{ tripleo_cephadm_nfs_rados_export_index }} + +NFSv4 { + RecoveryBackend = 'rados_kv'; +} +RADOS_KV { + ceph_conf = '/etc/ceph/{{ tripleo_cephadm_cluster }}.conf'; + userid = "{{ tripleo_cephadm_ceph_nfs_ceph_user }}"; + pool = "{{ cephfs_data_pool.name }}"; +} +{% endif %} + + +LOG { + Facility { + name = FILE; + destination = "/var/log/ganesha/ganesha.log"; + enable = active; + } +}