From 17e6e629f5660a364498b2bfd2f5aa6aede79859 Mon Sep 17 00:00:00 2001 From: xionglingfeng Date: Sat, 13 Feb 2016 23:50:46 -0330 Subject: [PATCH] Allow cinder-volume to be configured to use NFS Allow cinder-volume, nova-compute and nova-libvirtd to be configured to use NFS. In order to mount and work with NFS shares, several containers needed the NFS packages installed during build time. One somewhat significant change is the addition of an explicit bind volume for nova-compute that has shared mounts enabled. According to docker-run(1), the shared mount propagation flag can only be specified for bind mounted Docker volumes and not named volumes. In an NFS setup, cinder-volume mounts the NFS shares so that it can create and manage the Cinder volumes. When a new instance is created with a Cinder volume or a Cinder volume is attached to an existing instance, nova-compute mounts the Cinder volume from the NFS share for nova-libvirtd. In order for nova-libvirtd to then see those Cinder volumes the shared mounts flag must be enabled for the Docker volume. Remove the rpcbind container as it is only necessary for operators who are using NFSv3 or lower. There is no known need for this currently however, this container can be added in the future should an operator require it. Co-authored-by: Ryan Hallisey Co-authored-by: Andrew Widdersheim Change-Id: Iad77c05bce8876bdcc69b7ec22edd50e3bf48b9f Closes-Bug: #1530515 Partially implements: blueprint nfs-support-in-cinder --- ansible/group_vars/all.yml | 1 + ansible/roles/cinder/defaults/main.yml | 3 ++ ansible/roles/cinder/tasks/config.yml | 11 +++++++ .../cinder/templates/cinder-volume.json.j2 | 17 ++++++++--- ansible/roles/cinder/templates/cinder.conf.j2 | 7 +++++ ansible/roles/nova/tasks/start_compute.yml | 3 ++ docker/cinder/cinder-rpcbind/Dockerfile.j2 | 29 ------------------- docker/cinder/cinder-volume/Dockerfile.j2 | 5 ++-- docker/nova/nova-compute/Dockerfile.j2 | 4 +++ 9 files changed, 45 insertions(+), 35 deletions(-) delete mode 100644 docker/cinder/cinder-rpcbind/Dockerfile.j2 diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 24772f765c..66ddf8ff66 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -254,6 +254,7 @@ enable_ceph_rgw: "no" enable_cinder: "no" enable_cinder_backend_iscsi: "no" enable_cinder_backend_lvm: "no" +enable_cinder_backend_nfs: "no" enable_cloudkitty: "no" enable_congress: "no" enable_etcd: "no" diff --git a/ansible/roles/cinder/defaults/main.yml b/ansible/roles/cinder/defaults/main.yml index ad948e6a4a..757022f02c 100644 --- a/ansible/roles/cinder/defaults/main.yml +++ b/ansible/roles/cinder/defaults/main.yml @@ -74,5 +74,8 @@ cinder_backends: - name: "lvm-1" driver: "lvm" enabled: "{{ enable_cinder_backend_lvm | bool }}" + - name: "nfs-1" + driver: "nfs" + enabled: "{{ enable_cinder_backend_nfs | bool }}" cinder_enabled_backends: "{{ cinder_backends|selectattr('enabled', 'equalto', true)|list }}" diff --git a/ansible/roles/cinder/tasks/config.yml b/ansible/roles/cinder/tasks/config.yml index 800401615b..ecdd67c40e 100644 --- a/ansible/roles/cinder/tasks/config.yml +++ b/ansible/roles/cinder/tasks/config.yml @@ -49,3 +49,14 @@ dest: "{{ node_config_directory }}/cinder/policy.json" when: cinder_policy.stat.exists + +- name: Copying over nfs_shares files for cinder_volume + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/cinder-volume/nfs_shares" + with_first_found: + - "{{ node_custom_config }}/nfs_shares.j2" + - "{{ node_custom_config }}/cinder/nfs_shares.j2" + - "{{ node_custom_config }}/cinder/cinder-volume/nfs_shares.j2" + - "{{ node_custom_config }}/cinder/{{ inventory_hostname }}/nfs_shares.j2" + when: enable_cinder_backend_nfs | bool diff --git a/ansible/roles/cinder/templates/cinder-volume.json.j2 b/ansible/roles/cinder/templates/cinder-volume.json.j2 index db38911195..5dc729b145 100644 --- a/ansible/roles/cinder/templates/cinder-volume.json.j2 +++ b/ansible/roles/cinder/templates/cinder-volume.json.j2 @@ -6,19 +6,28 @@ "dest": "/etc/cinder/cinder.conf", "owner": "cinder", "perm": "0600" - }{% if cinder_backend_ceph | bool %}, + }, { "source": "{{ container_config_directory }}/ceph.*", "dest": "/etc/ceph/", "owner": "cinder", - "perm": "0700" + "perm": "0700", + "optional": {{ (not cinder_backend_ceph | bool) | string | lower }} }, { "source": "{{ container_config_directory }}/ceph.conf", "dest": "/etc/ceph/ceph.conf", "owner": "cinder", - "perm": "0600" - }{% endif %} + "perm": "0600", + "optional": {{ (not cinder_backend_ceph | bool) | string | lower }} + }, + { + "source": "{{ container_config_directory }}/nfs_shares", + "dest": "/etc/cinder/nfs_shares", + "owner": "cinder", + "perm": "0600", + "optional": {{ (not enable_cinder_backend_nfs | bool) | string | lower }} + } ], "permissions": [ { diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2 index a34ee75fd7..16f07b976c 100644 --- a/ansible/roles/cinder/templates/cinder.conf.j2 +++ b/ansible/roles/cinder/templates/cinder.conf.j2 @@ -99,5 +99,12 @@ rbd_secret_uuid = {{ rbd_secret_uuid }} report_discard_supported = True {% endif %} +{% if enable_cinder_backend_nfs | bool %} +[nfs-1] +volume_driver = cinder.volume.drivers.nfs.NfsDriver +volume_backend_name = nfs-1 +nfs_shares_config = /etc/cinder/nfs_shares +{% endif %} + [privsep_entrypoint] helper_command=sudo cinder-rootwrap /etc/cinder/rootwrap.conf privsep-helper --config-file /etc/cinder/cinder.conf diff --git a/ansible/roles/nova/tasks/start_compute.yml b/ansible/roles/nova/tasks/start_compute.yml index 1ba3c47c80..d40e7a007c 100644 --- a/ansible/roles/nova/tasks/start_compute.yml +++ b/ansible/roles/nova/tasks/start_compute.yml @@ -17,6 +17,7 @@ - "kolla_logs:/var/log/kolla/" - "libvirtd:/var/lib/libvirt" - "nova_compute:/var/lib/nova/" + - "/var/lib/nova/mnt:/var/lib/nova/mnt:shared" - "nova_libvirt_qemu:/etc/libvirt/qemu" register: start_nova_libvirt_container # NOTE(Jeffrey4l): retry 5 to remove nova_libvirt container because when @@ -37,6 +38,7 @@ - "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}" - "libvirtd:/var/lib/libvirt" - "nova_compute:/var/lib/nova/" + - "/var/lib/nova/mnt:/var/lib/nova/mnt:shared" - name: Starting nova-compute container kolla_docker: @@ -94,6 +96,7 @@ - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" - "nova_compute:/var/lib/nova" + - "/var/lib/nova/mnt:/var/lib/nova/mnt:shared" - "heka_socket:/var/lib/kolla/heka/" # TODO(jeffrey4l): how to handle the nova-compute-fake and # nova-compute-ironic diff --git a/docker/cinder/cinder-rpcbind/Dockerfile.j2 b/docker/cinder/cinder-rpcbind/Dockerfile.j2 deleted file mode 100644 index 60eb8672be..0000000000 --- a/docker/cinder/cinder-rpcbind/Dockerfile.j2 +++ /dev/null @@ -1,29 +0,0 @@ -FROM {{ namespace }}/{{ image_prefix }}cinder-base:{{ tag }} -MAINTAINER {{ maintainer }} - -{% block cinder_rpcbind_header %}{% endblock %} - -{% import "macros.j2" as macros with context %} - -{% if base_distro in ['centos', 'oraclelinux', 'rhel'] %} - - {% set cinder_rpcbind_packages = [ - 'nfs-utils', - 'nfs-utils-lib' - ] %} - -{% elif base_distro in ['ubuntu', 'debian'] %} - - {% set cinder_rpcbind_packages = [ - 'rpcbind' - ] %} - -{% endif %} - -{{ macros.install_packages(cinder_rpcbind_packages | customizable("packages")) }} - -{% block cinder_rpcbind_footer %}{% endblock %} -{% block footer %}{% endblock %} -{{ include_footer }} - -USER cinder diff --git a/docker/cinder/cinder-volume/Dockerfile.j2 b/docker/cinder/cinder-volume/Dockerfile.j2 index 11cd55c7ba..ef90fcd91a 100644 --- a/docker/cinder/cinder-volume/Dockerfile.j2 +++ b/docker/cinder/cinder-volume/Dockerfile.j2 @@ -8,6 +8,7 @@ MAINTAINER {{ maintainer }} {% if base_distro in ['centos', 'oraclelinux', 'rhel'] %} {% set cinder_volume_packages = [ + 'nfs-utils', 'scsi-target-utils' ] %} @@ -35,11 +36,11 @@ RUN sed -i '1 i include /var/lib/cinder/volumes/*' /etc/tgt/targets.conf {% if base_distro in ['ubuntu', 'debian'] %} {% set cinder_volume_packages = [ - 'cinder-volume' + 'cinder-volume', + 'nfs-common' ] %} {{ macros.install_packages(cinder_volume_packages | customizable("packages")) }} - {% endif %} {% endif %} diff --git a/docker/nova/nova-compute/Dockerfile.j2 b/docker/nova/nova-compute/Dockerfile.j2 index 461f8b1559..46e280a5f2 100644 --- a/docker/nova/nova-compute/Dockerfile.j2 +++ b/docker/nova/nova-compute/Dockerfile.j2 @@ -15,6 +15,7 @@ MAINTAINER {{ maintainer }} 'ceph-common', 'genisoimage', 'iscsi-initiator-utils', + 'nfs-utils', 'targetcli', 'python-rtslib' ] %} @@ -32,6 +33,7 @@ MAINTAINER {{ maintainer }} 'ironic-common', 'python-ironicclient', 'genisoimage', + 'nfs-common', 'open-iscsi', 'targetcli', 'python-rtslib' @@ -53,6 +55,7 @@ RUN rm -f /etc/nova/nova-compute.conf 'ceph-common', 'genisoimage', 'iscsi-initiator-utils', + 'nfs-utils', 'targetcli', 'python-rtslib' ] %} @@ -64,6 +67,7 @@ RUN rm -f /etc/nova/nova-compute.conf 'qemu-utils', 'ceph-common', 'genisoimage', + 'nfs-common', 'open-iscsi', 'targetcli', 'python-rtslib'