Browse Source

Create ganesha-export-index before the ceph-nfs daemon is deployed

When ceph_nfs_rados_backend is provided, as per doc [1], for a fresh
setup, we need to make sure to create the Ganesha export index object
as an empty object before starting the Ganesha server.
This change provides the two tasks, already present in ceph-ansible,
to create the index object.
In addition, since the ganesha systemd units are managed by pacemaker,
this change renders the related j2 templates and prepares the ganesha
workdir in the ceph-nfs target hosts.

[1] https://docs.openstack.org/manila/latest/contributor/ganesha.html
[2] https://github.com/ceph/ceph-ansible/blob/master/roles/ceph-nfs/tasks/start_nfs.yml#L15-L31

Change-Id: Ie89c35238d1bb82a28d560d7f0f718f7ee38044f
(cherry picked from commit 6415174661)
changes/15/792515/1
Francesco Pantano 2 months ago
parent
commit
ab58c38c03
8 changed files with 306 additions and 44 deletions
  1. +2
    -2
      tripleo_ansible/playbooks/cephadm.yml
  2. +2
    -0
      tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml
  3. +0
    -42
      tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml
  4. +34
    -0
      tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml
  5. +94
    -0
      tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml
  6. +78
    -0
      tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml
  7. +44
    -0
      tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2
  8. +52
    -0
      tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2

+ 2
- 2
tripleo_ansible/playbooks/cephadm.yml View File

@ -56,9 +56,9 @@
tasks_from: mds
- name: Config Ganesha
import_role:
include_role:
name: tripleo_cephadm
tasks_from: ganesha
tasks_from: nfs
when:
- cephfs_data_pool is defined
- cephfs_metadata_pool is defined


+ 2
- 0
tripleo_ansible/roles/tripleo_cephadm/defaults/main.yml View File

@ -45,5 +45,7 @@ tripleo_cephadm_fsid_list: []
tripleo_cephadm_fqdn: false
tripleo_cephadm_crush_rules: []
tripleo_cephadm_internal_tls_enabled: false
tripleo_cephadm_nfs_rados_export_index: 'ganesha-export-index'
tripleo_cephadm_ceph_nfs_rados_backend: true
# todo(fultonj) add is_hci boolean for target memory
# https://lists.ceph.io/hyperkitty/list/dev@ceph.io/thread/Z77XO23JPXDNHKM7IG6UN4URYKA6L7VH/

+ 0
- 42
tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha.yaml View File

@ -1,42 +0,0 @@
---
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Using two tasks with a boolean due to the ansible bug: https://github.com/ansible/ansible/issues/68364
- name: Collect the host and build the resulting host list
set_fact:
_hosts: "{{ _hosts|default([]) + [ hostvars[item].canonical_hostname ] }}"
with_items: "{{ groups['ceph_nfs'] }}"
when: tripleo_cephadm_fqdn | bool
- name: Collect the host and build the resulting host list
set_fact:
_hosts: "{{ _hosts|default([]) + [ hostvars[item].inventory_hostname ] }}"
with_items: "{{ groups['ceph_nfs'] }}"
when: not tripleo_cephadm_fqdn | bool
- name: Create the ganesha Daemon spec definition
become: true
ceph_mkspec:
service_type: nfs
apply: true
hosts: "{{ _hosts }}"
render_path: "{{ tripleo_cephadm_spec_home }}"
spec:
namespace: 'ganesha'
pool: "{{ cephfs_data_pool }}"
environment:
CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"

+ 34
- 0
tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/distribute_keys.yaml View File

@ -0,0 +1,34 @@
---
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: create keyring directory
file:
name: /var/lib/ceph/bootstrap-rgw/
state: directory
owner: "167"
group: "167"
mode: "0755"
delegate_to: "{{ nfs_node }}"
- name: copy ceph key(s) if needed
copy:
dest: "{{ item.item.path }}"
content: "{{ item.stdout + '\n' }}"
owner: "167"
group: "167"
mode: "0755"
with_items: "{{ _rgw_keys.results }}"
delegate_to: "{{ nfs_node }}"

+ 94
- 0
tripleo_ansible/roles/tripleo_cephadm/tasks/ganesha/start_nfs.yaml View File

@ -0,0 +1,94 @@
---
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: dbus related tasks
block:
- name: create dbus service file
copy:
content: "{{ dbus_ganesha_file.stdout }}"
dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf
owner: "root"
group: "root"
mode: "0644"
- name: reload dbus configuration
command: "killall -SIGHUP dbus-daemon"
when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool
delegate_to: "{{ nfs_node }}"
- name: Render config files
delegate_to: "{{ nfs_node }}"
block:
- name: Make sure /etc/ganesha exists
file:
path: /etc/ganesha
state: directory
owner: root
group: root
mode: "0755"
- name: create exports directory
file:
path: /etc/ganesha/export.d
state: directory
owner: "root"
group: "root"
mode: "0755"
when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool
- name: create exports dir index file
copy:
content: ""
force: false
dest: /etc/ganesha/export.d/INDEX.conf
owner: "root"
group: "root"
mode: "0644"
when: tripleo_cephadm_ceph_nfs_dynamic_exports | bool
- name: Render ganesha config file
template:
src: ganesha.conf.j2
dest: /etc/ganesha/ganesha.conf
become: true
- name: Render ganesha systemd unit
template:
src: ceph-nfs.service.j2
dest: /etc/systemd/system/ceph-nfs@.service
owner: "root"
group: "root"
mode: "0644"
become: true
- name: systemd start nfs container
systemd:
name: ceph-nfs@{{ tripleo_cephadm_ceph_nfs_service_suffix | default(ansible_facts['hostname']) }}
state: started
enabled: yes
masked: no
daemon_reload: yes
when:
- tripleo_cephadm_ceph_nfs_enable_service | bool
- name: start nfs gateway service
systemd:
name: nfs-ganesha
state: started
enabled: yes
masked: no
when:
- tripleo_cephadm_ceph_nfs_enable_service | bool

+ 78
- 0
tripleo_ansible/roles/tripleo_cephadm/tasks/nfs.yaml View File

@ -0,0 +1,78 @@
---
# Copyright 2021 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
- name: Get ceph_cli
include_tasks: ceph_cli.yaml
vars:
ceph_command: 'rados'
- name: check if rados index object exists
shell: |
{{ tripleo_cephadm_ceph_cli }} -p {{ cephfs_data_pool.name }} --cluster {{ tripleo_cephadm_cluster }} \
ls | grep {{ tripleo_cephadm_nfs_rados_export_index }}
changed_when: false
failed_when: false
register: rados_index_exists
check_mode: false
when: tripleo_cephadm_ceph_nfs_rados_backend | bool
- name: create an empty rados index object
command: |
{{ tripleo_cephadm_ceph_cli }} -p {{ cephfs_data_pool.name }} --cluster {{ tripleo_cephadm_cluster }} \
put {{ tripleo_cephadm_nfs_rados_export_index }} /dev/null
when:
- tripleo_cephadm_ceph_nfs_rados_backend | bool
- rados_index_exists.rc != 0
- name: Extract key
ceph_key:
name: "{{ item.name }}"
output_format: plain
state: info
environment:
CEPH_CONTAINER_IMAGE: "{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }}"
CEPH_CONTAINER_BINARY: "{{ tripleo_cephadm_container_cli }}"
register: _rgw_keys
become: true
with_items:
- { name: "client.bootstrap-rgw", path: "/var/lib/ceph/bootstrap-rgw/{{ tripleo_cephadm_cluster }}.keyring", copy_key: true }
- name: Distribute Ceph keys to the nfs nodes
include_tasks: ganesha/distribute_keys.yaml
vars:
nfs_node: "{{ node }}"
rgw_keys: "{{ _rgw_keys }}"
loop: "{{ groups['ceph_nfs'] }}"
loop_control:
loop_var: node
- name: get dbus-1 file
command: |
{{ container_cli }} run --rm --entrypoint=cat \
{{ tripleo_cephadm_container_ns + '/' + tripleo_cephadm_container_image + ':' + tripleo_cephadm_container_tag }} \
/etc/dbus-1/system.d/org.ganesha.nfsd.conf
register: _dbus_ganesha
run_once: true
changed_when: false
- name: Render Ganesha templates and files
include_tasks: ganesha/start_nfs.yaml
vars:
nfs_node: "{{ node }}"
dbus_ganesha_file: "{{ _dbus_ganesha }}"
loop: "{{ groups['ceph_nfs'] }}"
loop_control:
loop_var: node

+ 44
- 0
tripleo_ansible/roles/tripleo_cephadm/templates/ceph-nfs.service.j2 View File

@ -0,0 +1,44 @@
[Unit]
Description=NFS-Ganesha file server
Documentation=http://github.com/nfs-ganesha/nfs-ganesha/wiki
After=network.target
[Service]
EnvironmentFile=-/etc/environment
{% if container_cli == 'podman' %}
ExecStartPre=-/usr/bin/rm -f /%t/%n-pid /%t/%n-cid
ExecStartPre=-/usr/bin/{{ container_cli }} rm --storage ceph-nfs-%i
ExecStartPre=-/usr/bin/mkdir -p /var/log/ceph /var/log/ganesha
{% endif %}
ExecStartPre=-/usr/bin/{{ container_cli }} rm ceph-nfs-%i
ExecStartPre={{ '/bin/mkdir' if ansible_facts['os_family'] == 'Debian' else '/usr/bin/mkdir' }} -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha /var/log/ganesha
ExecStart=/usr/bin/{{ container_cli }} run --rm --net=host \
-d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \
-v /var/lib/ceph:/var/lib/ceph:z \
-v /etc/ceph:/etc/ceph:z \
-v /var/lib/nfs/ganesha:/var/lib/nfs/ganesha:z \
-v /etc/ganesha:/etc/ganesha:z \
-v /var/run/ceph:/var/run/ceph:z \
-v /var/log/ceph:/var/log/ceph:z \
-v /var/log/ganesha:/var/log/ganesha:z \
{% if tripleo_cephadm_ceph_nfs_dynamic_exports | bool %}
--privileged \
-v /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket \
{% endif -%}
-v /etc/localtime:/etc/localtime:ro \
-e CLUSTER={{ tripleo_cephadm_cluster }} \
-e CEPH_DAEMON=NFS \
-e CONTAINER_IMAGE={{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }} \
--name=ceph-nfs-{{ tripleo_cephadm_ceph_nfs_service_suffix | default(ansible_facts['hostname']) }} \
{{ tripleo_cephadm_container_ns }}/{{ tripleo_cephadm_container_image }}:{{ tripleo_cephadm_container_tag }}
ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_cli }} rm -f `cat /%t/%n-cid`"
KillMode=none
Restart=always
RestartSec=10s
TimeoutStartSec=120
TimeoutStopSec=15
Type=forking
PIDFile=/%t/%n-pid
[Install]
WantedBy=multi-user.target

+ 52
- 0
tripleo_ansible/roles/tripleo_cephadm/templates/ganesha.conf.j2 View File

@ -0,0 +1,52 @@
#jinja2: trim_blocks: "true", lstrip_blocks: "true"
# {{ ansible_managed }}
{% if tripleo_cephadm_ceph_nfs_dynamic_exports | bool and not tripleo_cephadm_ceph_nfs_rados_backend | bool %}
%include /etc/ganesha/export.d/INDEX.conf
{% endif %}
NFS_Core_Param
{
{% if tripleo_cephadm_ceph_nfs_bind_addr is defined %}
Bind_Addr={{ tripleo_cephadm_ceph_nfs_bind_addr }};
{% endif %}
}
{% if tripleo_cephadm_ceph_nfs_disable_caching | bool or nfs_file_gw | bool %}
EXPORT_DEFAULTS {
Attr_Expiration_Time = 0;
}
CACHEINODE {
Dir_Chunk = 0;
NParts = 1;
Cache_Size = 1;
}
{% endif %}
{% if tripleo_cephadm_ceph_nfs_rados_backend | bool %}
RADOS_URLS {
ceph_conf = '/etc/ceph/{{ tripleo_cephadm_cluster }}.conf';
userid = "{{ tripleo_cephadm_ceph_nfs_ceph_user }}";
}
%url rados://{{ cephfs_data_pool.name }}/{{ tripleo_cephadm_nfs_rados_export_index }}
NFSv4 {
RecoveryBackend = 'rados_kv';
}
RADOS_KV {
ceph_conf = '/etc/ceph/{{ tripleo_cephadm_cluster }}.conf';
userid = "{{ tripleo_cephadm_ceph_nfs_ceph_user }}";
pool = "{{ cephfs_data_pool.name }}";
}
{% endif %}
LOG {
Facility {
name = FILE;
destination = "/var/log/ganesha/ganesha.log";
enable = active;
}
}

Loading…
Cancel
Save