External Ceph - Implementation Cinder

This patch adds support for external Ceph clusters for Cinder.

For clean integration the backend configuration mechanism had to be
slightly adjusted.

We now have the option to enable multiple backends for Cinder
independently.

Currently, the flags cinder_backend_iscsi and cinder_backend_ceph are
used to toggle backends.

Documentation on how to use external ceph was added.

Change-Id: I7e0267b90d62d6d881f24f063cdb894422ec8618
Partially-Implements: Blueprint: external-ceph

ansible/roles/cinder/tasks/bootstrap.yml

@@ -45,11 +45,11 @@
     action: "create_volume"
     common_options: "{{ docker_common_options }}"
     name: "cinder"
-  when: "{{ enable_iscsi | bool }}"
+  when: "{{ cinder_backend_iscsi | bool }}"
 
 - name: Creating iscsi info volume
   kolla_docker:
     action: "create_volume"
     common_options: "{{ docker_common_options }}"
     name: "iscsi_info"
-  when: "{{ enable_iscsi | bool }}"
+  when: "{{ cinder_backend_iscsi | bool }}"

ansible/roles/cinder/tasks/deploy.yml

@@ -1,13 +1,19 @@
 ---
 - include: ceph.yml
   when:
-    - enable_ceph | bool
+    - (enable_ceph | bool) and (cinder_backend_ceph | bool)
     - inventory_hostname in groups['ceph-mon'] or
         inventory_hostname in groups['cinder-api'] or
         inventory_hostname in groups['cinder-volume'] or
         inventory_hostname in groups['cinder-scheduler'] or
         inventory_hostname in groups['cinder-backup']
 
+- include: external_ceph.yml
+  when:
+    - (enable_ceph | bool == False) and (cinder_backend_ceph | bool)
+    - inventory_hostname in groups['cinder-volume'] or
+      inventory_hostname in groups['cinder-backup']
+
 - include: register.yml
   when: inventory_hostname in groups['cinder-api']
 

ansible/roles/cinder/tasks/external_ceph.yml

@@ -0,0 +1,36 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/cinder-{{ item }}"
+    state: "directory"
+  when: inventory_hostname in groups['cinder-volume'] or
+        inventory_hostname in groups['cinder-backup']
+  with_items:
+    - volume
+    - backup
+
+- name: Copying over ceph.conf for Cinder
+  merge_configs:
+    vars:
+      service_name: "{{ item }}"
+    sources:
+      - "{{ node_custom_config }}/cinder/ceph.conf"
+      - "{{ node_custom_config }}/cinder/{{ item }}/ceph.conf"
+    dest: "{{ node_config_directory }}/{{ item }}/ceph.conf"
+  with_items:
+    - "cinder-backup"
+    - "cinder-volume"
+
+- name: Copy over Ceph keyring files for cinder-volume
+  copy:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/cinder-volume/"
+  with_fileglob:
+    - "{{ node_custom_config }}/cinder/cinder-volume/ceph.client*"
+
+- name: Copy over Ceph keyring files for cinder-backup
+  copy:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/cinder-backup/"
+  with_fileglob:
+    - "{{ node_custom_config }}/cinder/cinder-volume/ceph.client*"

ansible/roles/cinder/tasks/start.yml

@@ -30,8 +30,8 @@
       - "/etc/localtime:/etc/localtime:ro"
       - "/dev/:/dev/"
       - "/run/:/run/"
-      - "{% if enable_iscsi | bool %}cinder:/var/lib/cinder{% endif %}"
-      - "{% if enable_iscsi | bool %}iscsi_info:/etc/iscsi{% endif %}"
+      - "{% if cinder_backend_iscsi | bool %}cinder:/var/lib/cinder{% endif %}"
+      - "{% if cinder_backend_iscsi | bool %}iscsi_info:/etc/iscsi{% endif %}"
       - "kolla_logs:/var/log/kolla/"
 
 - name: Starting cinder-volume container

ansible/roles/cinder/templates/cinder-backup.json.j2

@@ -6,27 +6,13 @@
             "dest": "/etc/cinder/cinder.conf",
             "owner": "cinder",
             "perm": "0600"
-        },
+        }{% if cinder_backend_ceph | bool %},
         {
-            "source": "{{ container_config_directory }}/ceph.client.cinder-backup.keyring",
-            "dest": "/etc/ceph/ceph.client.cinder-backup.keyring",
+            "source": "{{ container_config_directory }}/ceph.*",
+            "dest": "/etc/ceph/",
             "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
-        },
-        {
-            "source": "{{ container_config_directory }}/ceph.client.cinder.keyring",
-            "dest": "/etc/ceph/ceph.client.cinder.keyring",
-            "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
-        },
-        {
-            "source": "{{ container_config_directory }}/ceph.conf",
-            "dest": "/etc/ceph/ceph.conf",
-            "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
-        }
+            "perm": "0700",
+            "optional": {{ (not cinder_backend_ceph | bool) | string | lower }}
+        }{% endif %}
     ]
 }

ansible/roles/cinder/templates/cinder-volume.json.j2

@@ -6,20 +6,18 @@
             "dest": "/etc/cinder/cinder.conf",
             "owner": "cinder",
             "perm": "0600"
-        },
+        }{% if cinder_backend_ceph | bool %},
         {
-            "source": "{{ container_config_directory }}/ceph.client.cinder.keyring",
-            "dest": "/etc/ceph/ceph.client.cinder.keyring",
+            "source": "{{ container_config_directory }}/ceph.*",
+            "dest": "/etc/ceph/",
             "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
+            "perm": "0700"
         },
         {
             "source": "{{ container_config_directory }}/ceph.conf",
             "dest": "/etc/ceph/ceph.conf",
             "owner": "cinder",
-            "perm": "0600",
-            "optional": {{ (not enable_ceph | bool) | string | lower }}
-        }
+            "perm": "0600"
+        }{% endif %}
     ]
 }

ansible/roles/cinder/templates/cinder.conf.j2

@@ -16,14 +16,14 @@ glance_api_version = 2
 
 os_region_name = {{ openstack_region_name }}
 
-{% if cinder_volume_driver == "lvm" %}
-enabled_backends = {{ cinder_volume_backend_name }}
-{% elif cinder_volume_driver == "ceph" %}
+{% if enable_ceph | bool and cinder_backend_ceph | bool %}
 default_volume_type = rbd-1
 enabled_backends = rbd-1
+{% elif cinder_backend_iscsi | bool %}
+enabled_backends = {{ cinder_volume_backend_name }}
 {% endif %}
 
-{% if service_name == "cinder-backup" and cinder_volume_driver == "ceph" %}
+{% if service_name == "cinder-backup" and enable_ceph | bool and cinder_backend_ceph | bool %}
 backup_driver = cinder.backup.drivers.ceph
 backup_ceph_conf = /etc/ceph/ceph.conf
 backup_ceph_user = cinder-backup
@@ -76,7 +76,7 @@ rabbit_ha_queues = true
 rabbit_hosts = {% for host in groups['rabbitmq'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}
 
 
-{% if cinder_volume_driver == "lvm" %}
+{% if cinder_backend_iscsi | bool %}
 [{{ cinder_volume_backend_name }}]
 volume_group = {{ cinder_volume_group }}
 volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver
@@ -84,7 +84,8 @@ volume_backend_name = {{ cinder_volume_backend_name }}
 iscsi_ip_address = {{ hostvars[groups['tgtd'][0]]['ansible_' + hostvars[groups['tgtd'][0]]['storage_interface']]['ipv4']['address'] }}
 iscsi_helper = {{ cinder_iscsi_helper }}
 iscsi_protocol = {{ cinder_iscsi_protocol }}
-{% elif cinder_volume_driver == "ceph" %}
+
+{% elif enable_ceph | bool and cinder_backend_ceph | bool %}
 [rbd-1]
 volume_driver = cinder.volume.drivers.rbd.RBDDriver
 rbd_pool = {{ ceph_cinder_pool_name }}