diff --git a/tripleo_ansible/playbooks/cephadm.yml b/tripleo_ansible/playbooks/cephadm.yml index 94d4e8bbd..c2100b54a 100644 --- a/tripleo_ansible/playbooks/cephadm.yml +++ b/tripleo_ansible/playbooks/cephadm.yml @@ -56,9 +56,9 @@ tasks_from: mds - name: Config Ganesha - import_role: + include_role: name: tripleo_cephadm - tasks_from: ganesha + tasks_from: nfs when: - cephfs_data_pool is defined - cephfs_metadata_pool is defined diff --git a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml index 9cbeda469..1f112e985 100644 --- a/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml +++ b/tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml @@ -45,5 +45,7 @@ tripleo_cephadm_fsid_list: [] tripleo_cephadm_fqdn: false tripleo_cephadm_crush_rules: [] tripleo_cephadm_internal_tls_enabled: false +tripleo_cephadm_nfs_rados_export_index: 'ganesha-export-index' +tripleo_cephadm_ceph_nfs_rados_backend: true # todo(fultonj) add is_hci boolean for target memory # https://lists.ceph.io/hyperkitty/list/dev@ceph.io/thread/Z77XO23JPXDNHKM7IG6UN4URYKA6L7VH/ diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml deleted file mode 100644 index 842e40ce1..000000000 --- a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -# Copyright 2021 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Using two tasks with a boolean due to the ansible bug: https://github.com/ansible/ansible/issues/68364 -- name: Collect the host and build the resulting host list - set_fact: - _hosts: "{{ _hosts|default([]) + [ hostvars[item].canonical_hostname ] }}" - with_items: "{{ groups['ceph_nfs'] }}" - when: tripleo_cephadm_fqdn | bool - -- name: Collect the host and build the resulting host list - set_fact: - _hosts: "{{ _hosts|default([]) + [ hostvars[item].inventory_hostname ] }}" - with_items: "{{ groups['ceph_nfs'] }}" - when: not tripleo_cephadm_fqdn | bool - -- name: Create the ganesha Daemon spec definition - become: true - ceph_mkspec: - service_type: nfs - apply: true - hosts: "{{ _hosts }}" - render_path: "{{ tripleo_cephadm_spec_home }}" - spec: - namespace: 'ganesha' - pool: "{{ cephfs_data_pool }}" - environment: - CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" - CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml new file mode 100644 index 000000000..b83c995c8 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml @@ -0,0 +1,34 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: create keyring directory + file: + name: /var/lib/ceph/bootstrap-rgw/ + state: directory + owner: "167" + group: "167" + mode: "0755" + delegate_to: "{{ nfs_node }}" + +- name: copy ceph key(s) if needed + copy: + dest: "{{ item.item.path }}" + content: "{{ item.stdout + '\n' }}" + owner: "167" + group: "167" + mode: "0755" + with_items: "{{ _rgw_keys.results }}" + delegate_to: "{{ nfs_node }}" diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml new file mode 100644 index 000000000..af40e4fb8 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml @@ -0,0 +1,94 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: dbus related tasks + block: + - name: create dbus service file + copy: + content: "{{ dbus_ganesha_file.stdout }}" + dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf + owner: "root" + group: "root" + mode: "0644" + + - name: reload dbus configuration + command: "killall -SIGHUP dbus-daemon" + when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool + delegate_to: "{{ nfs_node }}" + +- name: Render config files + delegate_to: "{{ nfs_node }}" + block: + - name: Make sure /etc/ganesha exists + file: + path: /etc/ganesha + state: directory + owner: root + group: root + mode: "0755" + + - name: create exports directory + file: + path: /etc/ganesha/export.d + state: directory + owner: "root" + group: "root" + mode: "0755" + when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool + + - name: create exports dir index file + copy: + content: "" + force: false + dest: /etc/ganesha/export.d/INDEX.conf + owner: "root" + group: "root" + mode: "0644" + when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool + + - name: Render ganesha config file + template: + src: ganesha.conf.j2 + dest: /etc/ganesha/ganesha.conf + become: true + + - name: Render ganesha systemd unit + template: + src: ceph-nfs.service.j2 + dest: /etc/systemd/system/ceph-nfs@.service + owner: "root" + group: "root" + mode: "0644" + become: true + + - name: systemd start nfs container + systemd: + name: ceph-nfs@{{ tripleo_cephadm_ceph_nfs_service_suffix | default(ansible_facts['hostname']) }} + state: started + enabled: yes + masked: no + daemon_reload: yes + when: + - tripleo_cephadm_ceph_nfs_enable_service | bool + + - name: start nfs gateway service + systemd: + name: nfs-ganesha + state: started + enabled: yes + masked: no + when: + - tripleo_cephadm_ceph_nfs_enable_service | bool diff --git a/tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml b/tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml new file mode 100644 index 000000000..9ed1344ef --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml @@ -0,0 +1,78 @@ +--- +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Get ceph_cli + include_tasks: ceph_cli.yaml + vars: + ceph_command: 'rados' + +- name: check if rados index object exists + shell: | + {{ tripleo_cephadm_ceph_cli }} -p {{ cephfs_data_pool.name }} --cluster {{ tripleo_cephadm_cluster }} \ + ls | grep {{ tripleo_cephadm_nfs_rados_export_index }} + changed_when: false + failed_when: false + register: rados_index_exists + check_mode: false + when: tripleo_cephadm_ceph_nfs_rados_backend | bool + +- name: create an empty rados index object + command: | + {{ tripleo_cephadm_ceph_cli }} -p {{ cephfs_data_pool.name }} --cluster {{ tripleo_cephadm_cluster }} \ + put {{ tripleo_cephadm_nfs_rados_export_index }} /dev/null + when: + - tripleo_cephadm_ceph_nfs_rados_backend | bool + - rados_index_exists.rc != 0 + +- name: Extract key + ceph_key: + name: "{{ item.name }}" + output_format: plain + state: info + environment: + CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}" + CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}" + register: _rgw_keys + become: true + with_items: + - { name: "client.bootstrap-rgw", path: "/var/lib/ceph/bootstrap-rgw/{{ tripleo_cephadm_cluster }}.keyring", copy_key: true } + +- name: Distribute Ceph keys to the nfs nodes + include_tasks: ganesha/distribute_keys.yaml + vars: + nfs_node: "{{ node }}" + rgw_keys: "{{ _rgw_keys }}" + loop: "{{ groups['ceph_nfs'] }}" + loop_control: + loop_var: node + +- name: get dbus-1 file + command: | + {{ container_cli }} run --rm --entrypoint=cat \ + {{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }} \ + /etc/dbus-1/system.d/org.ganesha.nfsd.conf + register: _dbus_ganesha + run_once: true + changed_when: false + +- name: Render Ganesha templates and files + include_tasks: ganesha/start_nfs.yaml + vars: + nfs_node: "{{ node }}" + dbus_ganesha_file: "{{ _dbus_ganesha }}" + loop: "{{ groups['ceph_nfs'] }}" + loop_control: + loop_var: node diff --git a/tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2 b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2 new file mode 100644 index 000000000..6414cc17e --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2 @@ -0,0 +1,44 @@ +[Unit] +Description=NFS-Ganesha file server +Documentation=http://github.com/nfs-ganesha/nfs-ganesha/wiki +After=network.target + +[Service] +EnvironmentFile=-/etc/environment +{% if container_cli == 'podman' %} +ExecStartPre=-/usr/bin/rm -f /%t/%n-pid /%t/%n-cid +ExecStartPre=-/usr/bin/{{ container_cli }} rm --storage ceph-nfs-%i +ExecStartPre=-/usr/bin/mkdir -p /var/log/ceph /var/log/ganesha +{% endif %} +ExecStartPre=-/usr/bin/{{ container_cli }} rm ceph-nfs-%i +ExecStartPre={{ '/bin/mkdir' if ansible_facts['os_family'] == 'Debian' else '/usr/bin/mkdir' }} -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha /var/log/ganesha +ExecStart=/usr/bin/{{ container_cli }} run --rm --net=host \ + -d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \ + -v /var/lib/ceph:/var/lib/ceph:z \ + -v /etc/ceph:/etc/ceph:z \ + -v /var/lib/nfs/ganesha:/var/lib/nfs/ganesha:z \ + -v /etc/ganesha:/etc/ganesha:z \ + -v /var/run/ceph:/var/run/ceph:z \ + -v /var/log/ceph:/var/log/ceph:z \ + -v /var/log/ganesha:/var/log/ganesha:z \ + {% if tripleo_cephadm_ceph_nfs_dynamic_exports | bool %} + --privileged \ + -v /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket \ + {% endif -%} + -v /etc/localtime:/etc/localtime:ro \ + -e CLUSTER={{ tripleo_cephadm_cluster }} \ + -e CEPH_DAEMON=NFS \ + -e CONTAINER_IMAGE={{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} \ + --name=ceph-nfs-{{ tripleo_cephadm_ceph_nfs_service_suffix | default(ansible_facts['hostname']) }} \ + {{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} +ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_cli }} rm -f `cat /%t/%n-cid`" +KillMode=none +Restart=always +RestartSec=10s +TimeoutStartSec=120 +TimeoutStopSec=15 +Type=forking +PIDFile=/%t/%n-pid + +[Install] +WantedBy=multi-user.target diff --git a/tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2 b/tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2 new file mode 100644 index 000000000..54fce8853 --- /dev/null +++ b/tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2 @@ -0,0 +1,52 @@ +#jinja2: trim_blocks: "true", lstrip_blocks: "true" +# {{ ansible_managed }} + +{% if tripleo_cephadm_ceph_nfs_dynamic_exports | bool and not tripleo_cephadm_ceph_nfs_rados_backend | bool %} +%include /etc/ganesha/export.d/INDEX.conf +{% endif %} + +NFS_Core_Param +{ +{% if tripleo_cephadm_ceph_nfs_bind_addr is defined %} + Bind_Addr={{ tripleo_cephadm_ceph_nfs_bind_addr }}; +{% endif %} +} + +{% if tripleo_cephadm_ceph_nfs_disable_caching | bool or nfs_file_gw | bool %} +EXPORT_DEFAULTS { + Attr_Expiration_Time = 0; +} + +CACHEINODE { + Dir_Chunk = 0; + + NParts = 1; + Cache_Size = 1; +} +{% endif %} + +{% if tripleo_cephadm_ceph_nfs_rados_backend | bool %} +RADOS_URLS { + ceph_conf = '/etc/ceph/{{ tripleo_cephadm_cluster }}.conf'; + userid = "{{ tripleo_cephadm_ceph_nfs_ceph_user }}"; +} +%url rados://{{ cephfs_data_pool.name }}/{{ tripleo_cephadm_nfs_rados_export_index }} + +NFSv4 { + RecoveryBackend = 'rados_kv'; +} +RADOS_KV { + ceph_conf = '/etc/ceph/{{ tripleo_cephadm_cluster }}.conf'; + userid = "{{ tripleo_cephadm_ceph_nfs_ceph_user }}"; + pool = "{{ cephfs_data_pool.name }}"; +} +{% endif %} + + +LOG { + Facility { + name = FILE; + destination = "/var/log/ganesha/ganesha.log"; + enable = active; + } +}