Create ganesha-export-index before the ceph-nfs daemon is deployed

When ceph_nfs_rados_backend is provided, as per doc [1], for a fresh
setup, we need to make sure to create the Ganesha export index object
as an empty object before starting the Ganesha server.
This change provides the two tasks, already present in ceph-ansible,
to create the index object.
In addition, since the ganesha systemd units are managed by pacemaker,
this change renders the related j2 templates and prepares the ganesha
workdir in the ceph-nfs target hosts.

[1] https://docs.openstack.org/manila/latest/contributor/ganesha.html
[2] https://github.com/ceph/ceph-ansible/blob/master/roles/ceph-nfs/tasks/start_nfs.yml#L15-L31

Change-Id: Ie89c35238d1bb82a28d560d7f0f718f7ee38044f

tripleo_ansible/playbooks/cephadm.yml

@@ -56,9 +56,9 @@
             tasks_from: mds
 
         - name: Config Ganesha
-          import_role:
+          include_role:
             name: tripleo_cephadm
-            tasks_from: ganesha
+            tasks_from: nfs
       when:
         - cephfs_data_pool is defined
         - cephfs_metadata_pool is defined

tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml

@@ -45,5 +45,7 @@ tripleo_cephadm_fsid_list: []
 tripleo_cephadm_fqdn: false
 tripleo_cephadm_crush_rules: []
 tripleo_cephadm_internal_tls_enabled: false
+tripleo_cephadm_nfs_rados_export_index: 'ganesha-export-index'
+tripleo_cephadm_ceph_nfs_rados_backend: true
 # todo(fultonj) add is_hci boolean for target memory
 # https://lists.ceph.io/hyperkitty/list/dev@ceph.io/thread/Z77XO23JPXDNHKM7IG6UN4URYKA6L7VH/

tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml

@@ -1,42 +0,0 @@
----
-# Copyright 2021 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# Using two tasks with a boolean due to the ansible bug: https://github.com/ansible/ansible/issues/68364
-- name: Collect the host and build the resulting host list
-  set_fact:
-    _hosts: "{{ _hosts|default([]) + [ hostvars[item].canonical_hostname ] }}"
-  with_items: "{{ groups['ceph_nfs'] }}"
-  when: tripleo_cephadm_fqdn | bool
-
-- name: Collect the host and build the resulting host list
-  set_fact:
-    _hosts: "{{ _hosts|default([]) + [ hostvars[item].inventory_hostname ] }}"
-  with_items: "{{ groups['ceph_nfs'] }}"
-  when: not tripleo_cephadm_fqdn | bool
-
-- name: Create the ganesha Daemon spec definition
-  become: true
-  ceph_mkspec:
-    service_type: nfs
-    apply: true
-    hosts: "{{ _hosts }}"
-    render_path: "{{ tripleo_cephadm_spec_home }}"
-    spec:
-      namespace: 'ganesha'
-      pool: "{{ cephfs_data_pool }}"
-  environment:
-    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
-    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"

tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml

@@ -0,0 +1,34 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: create keyring directory
+  file:
+    name: /var/lib/ceph/bootstrap-rgw/
+    state: directory
+    owner: "167"
+    group: "167"
+    mode: "0755"
+  delegate_to: "{{ nfs_node }}"
+
+- name: copy ceph key(s) if needed
+  copy:
+    dest: "{{ item.item.path }}"
+    content: "{{ item.stdout + '\n' }}"
+    owner: "167"
+    group: "167"
+    mode: "0755"
+  with_items: "{{ _rgw_keys.results }}"
+  delegate_to: "{{ nfs_node }}"

tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml

@@ -0,0 +1,94 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: dbus related tasks
+  block:
+    - name: create dbus service file
+      copy:
+        content: "{{ dbus_ganesha_file.stdout }}"
+        dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf
+        owner: "root"
+        group: "root"
+        mode: "0644"
+
+    - name: reload dbus configuration
+      command: "killall -SIGHUP dbus-daemon"
+  when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool
+  delegate_to: "{{ nfs_node }}"
+
+- name: Render config files
+  delegate_to: "{{ nfs_node }}"
+  block:
+    - name: Make sure /etc/ganesha exists
+      file:
+        path: /etc/ganesha
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: create exports directory
+      file:
+        path: /etc/ganesha/export.d
+        state: directory
+        owner: "root"
+        group: "root"
+        mode: "0755"
+      when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool
+
+    - name: create exports dir index file
+      copy:
+        content: ""
+        force: false
+        dest: /etc/ganesha/export.d/INDEX.conf
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool
+
+    - name: Render ganesha config file
+      template:
+        src: ganesha.conf.j2
+        dest: /etc/ganesha/ganesha.conf
+      become: true
+
+    - name: Render ganesha systemd unit
+      template:
+        src: ceph-nfs.service.j2
+        dest: /etc/systemd/system/ceph-nfs@.service
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      become: true
+
+    - name: systemd start nfs container
+      systemd:
+        name: ceph-nfs@{{ tripleo_cephadm_ceph_nfs_service_suffix | default(ansible_facts['hostname']) }}
+        state: started
+        enabled: yes
+        masked: no
+        daemon_reload: yes
+      when:
+        - tripleo_cephadm_ceph_nfs_enable_service | bool
+
+    - name: start nfs gateway service
+      systemd:
+        name: nfs-ganesha
+        state: started
+        enabled: yes
+        masked: no
+      when:
+        - tripleo_cephadm_ceph_nfs_enable_service | bool

tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml

@@ -0,0 +1,78 @@
+---
+# Copyright 2021 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Get ceph_cli
+  include_tasks: ceph_cli.yaml
+  vars:
+    ceph_command: 'rados'
+
+- name: check if rados index object exists
+  shell: |
+    {{ tripleo_cephadm_ceph_cli }} -p {{ cephfs_data_pool.name }} --cluster {{ tripleo_cephadm_cluster }} \
+    ls | grep {{ tripleo_cephadm_nfs_rados_export_index }}
+  changed_when: false
+  failed_when: false
+  register: rados_index_exists
+  check_mode: false
+  when: tripleo_cephadm_ceph_nfs_rados_backend | bool
+
+- name: create an empty rados index object
+  command: |
+    {{ tripleo_cephadm_ceph_cli }} -p {{ cephfs_data_pool.name }} --cluster {{ tripleo_cephadm_cluster }} \
+    put {{ tripleo_cephadm_nfs_rados_export_index }} /dev/null
+  when:
+    - tripleo_cephadm_ceph_nfs_rados_backend | bool
+    - rados_index_exists.rc != 0
+
+- name: Extract key
+  ceph_key:
+    name: "{{ item.name }}"
+    output_format: plain
+    state: info
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
+    CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
+  register: _rgw_keys
+  become: true
+  with_items:
+    - { name: "client.bootstrap-rgw", path: "/var/lib/ceph/bootstrap-rgw/{{ tripleo_cephadm_cluster }}.keyring", copy_key: true }
+
+- name: Distribute Ceph keys to the nfs nodes
+  include_tasks: ganesha/distribute_keys.yaml
+  vars:
+    nfs_node: "{{ node }}"
+    rgw_keys: "{{ _rgw_keys }}"
+  loop: "{{ groups['ceph_nfs'] }}"
+  loop_control:
+    loop_var: node
+
+- name: get dbus-1 file
+  command: |
+    {{ container_cli }} run --rm --entrypoint=cat \
+    {{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }} \
+    /etc/dbus-1/system.d/org.ganesha.nfsd.conf
+  register: _dbus_ganesha
+  run_once: true
+  changed_when: false
+
+- name: Render Ganesha templates and files
+  include_tasks: ganesha/start_nfs.yaml
+  vars:
+    nfs_node: "{{ node }}"
+    dbus_ganesha_file: "{{ _dbus_ganesha }}"
+  loop: "{{ groups['ceph_nfs'] }}"
+  loop_control:
+    loop_var: node

tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2

@@ -0,0 +1,44 @@
+[Unit]
+Description=NFS-Ganesha file server
+Documentation=http://github.com/nfs-ganesha/nfs-ganesha/wiki
+After=network.target
+
+[Service]
+EnvironmentFile=-/etc/environment
+{% if container_cli == 'podman' %}
+ExecStartPre=-/usr/bin/rm -f /%t/%n-pid /%t/%n-cid
+ExecStartPre=-/usr/bin/{{ container_cli }} rm --storage ceph-nfs-%i
+ExecStartPre=-/usr/bin/mkdir -p /var/log/ceph /var/log/ganesha
+{% endif %}
+ExecStartPre=-/usr/bin/{{ container_cli }} rm ceph-nfs-%i
+ExecStartPre={{ '/bin/mkdir' if ansible_facts['os_family'] == 'Debian' else '/usr/bin/mkdir' }} -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha /var/log/ganesha
+ExecStart=/usr/bin/{{ container_cli }} run --rm --net=host \
+  -d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \
+  -v /var/lib/ceph:/var/lib/ceph:z \
+  -v /etc/ceph:/etc/ceph:z \
+  -v /var/lib/nfs/ganesha:/var/lib/nfs/ganesha:z \
+  -v /etc/ganesha:/etc/ganesha:z \
+  -v /var/run/ceph:/var/run/ceph:z \
+  -v /var/log/ceph:/var/log/ceph:z \
+  -v /var/log/ganesha:/var/log/ganesha:z \
+  {% if tripleo_cephadm_ceph_nfs_dynamic_exports | bool %}
+  --privileged \
+  -v /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket \
+  {% endif -%}
+  -v /etc/localtime:/etc/localtime:ro \
+  -e CLUSTER={{ tripleo_cephadm_cluster }} \
+  -e CEPH_DAEMON=NFS \
+  -e CONTAINER_IMAGE={{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} \
+  --name=ceph-nfs-{{ tripleo_cephadm_ceph_nfs_service_suffix | default(ansible_facts['hostname']) }} \
+  {{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }}
+ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_cli }} rm -f `cat /%t/%n-cid`"
+KillMode=none
+Restart=always
+RestartSec=10s
+TimeoutStartSec=120
+TimeoutStopSec=15
+Type=forking
+PIDFile=/%t/%n-pid
+
+[Install]
+WantedBy=multi-user.target

tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2

@@ -0,0 +1,52 @@
+#jinja2: trim_blocks: "true", lstrip_blocks: "true"
+# {{ ansible_managed }}
+
+{% if tripleo_cephadm_ceph_nfs_dynamic_exports | bool and not tripleo_cephadm_ceph_nfs_rados_backend | bool %}
+%include /etc/ganesha/export.d/INDEX.conf
+{% endif %}
+
+NFS_Core_Param
+{
+{% if tripleo_cephadm_ceph_nfs_bind_addr is defined %}
+       Bind_Addr={{ tripleo_cephadm_ceph_nfs_bind_addr }};
+{% endif %}
+}
+
+{% if tripleo_cephadm_ceph_nfs_disable_caching | bool or nfs_file_gw | bool %}
+EXPORT_DEFAULTS {
+        Attr_Expiration_Time = 0;
+}
+
+CACHEINODE {
+        Dir_Chunk = 0;
+
+        NParts = 1;
+        Cache_Size = 1;
+}
+{% endif %}
+
+{% if tripleo_cephadm_ceph_nfs_rados_backend | bool %}
+RADOS_URLS {
+   ceph_conf = '/etc/ceph/{{ tripleo_cephadm_cluster }}.conf';
+   userid = "{{ tripleo_cephadm_ceph_nfs_ceph_user }}";
+}
+%url rados://{{ cephfs_data_pool.name }}/{{ tripleo_cephadm_nfs_rados_export_index }}
+
+NFSv4 {
+        RecoveryBackend = 'rados_kv';
+}
+RADOS_KV {
+        ceph_conf = '/etc/ceph/{{ tripleo_cephadm_cluster }}.conf';
+        userid = "{{ tripleo_cephadm_ceph_nfs_ceph_user }}";
+        pool = "{{ cephfs_data_pool.name }}";
+}
+{% endif %}
+
+
+LOG {
+        Facility {
+                name = FILE;
+                destination = "/var/log/ganesha/ganesha.log";
+                enable = active;
+        }
+}