External Ceph - Implementation Nova

Introduced nova backend selection flag for Ceph and priority if
multiple backends are configured

Add mechanism to deploy arbitrary ceph.conf and keyring files into
nova-compute and nova-libvirt containers

Added documentation

Change-Id: Id010ca9cc2d914e5358ef79edeb600a28220dd4b
Implements: blueprint external-ceph

ansible/group_vars/all.yml

@@ -254,6 +254,13 @@ cinder_volume_backend_name: "{{ 'cinder-volumes' if cinder_backend_iscsi | bool
 cinder_iscsi_helper: "{{ 'tgtadm' if cinder_backend_iscsi | bool else '' }}"
 cinder_iscsi_protocol: "{{ 'iscsi' if cinder_backend_iscsi | bool else '' }}"
 
+
+#######################
+# Nova options
+#######################
+nova_backend_ceph: "{{ enable_ceph }}"
+nova_backend: "{{ 'rbd' if nova_backend_ceph | bool else 'default' }}"
+
 ###################
 # Ceph options
 ###################

ansible/roles/nova/tasks/deploy.yml

@@ -1,7 +1,7 @@
 ---
 - include: ceph.yml
   when:
-    - enable_ceph | bool
+    - enable_ceph | bool and nova_backend == "rbd"
     - inventory_hostname in groups['ceph-mon'] or
         inventory_hostname in groups['compute'] or
         inventory_hostname in groups['nova-api'] or
@@ -10,6 +10,11 @@
         inventory_hostname in groups['nova-novncproxy'] or
         inventory_hostname in groups['nova-scheduler']
 
+- include: external-ceph.yml
+  when:
+    - enable_ceph | bool == False and nova_backend == "rbd"
+    - inventory_hostname in groups['compute']
+
 - include: register.yml
   when: inventory_hostname in groups['nova-api']
 

ansible/roles/nova/tasks/external-ceph.yml

@@ -0,0 +1,49 @@
+---
+- name: Ensuring config directory exists
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+  with_items:
+    - "nova-compute"
+    - "nova-libvirt/secrets"
+  when: inventory_hostname in groups['compute']
+
+- name: Find keyring files
+  local_action: find paths="{{ node_custom_config }}/nova/" patterns="^ceph\.client\..*?\.keyring$" use_regex=True
+  register: cephx_keyring_files
+
+- name: Copy over ceph keyring file
+  copy:
+    src: "{{ cephx_keyring_files.files[0].path }}"
+    dest: "{{ node_config_directory }}/{{item}}/"
+  with_items:
+    - nova-compute
+    - nova-libvirt
+  when: inventory_hostname in groups['compute']
+
+- name: Copy over ceph.conf
+  copy:
+    src: "{{ node_custom_config }}/nova/ceph.conf"
+    dest: "{{ node_config_directory }}/{{ item }}/"
+  with_items:
+    - nova-compute
+    - nova-libvirt
+  when: inventory_hostname in groups['compute']
+
+- name: Pushing secrets xml for libvirt
+  template:
+    src: "secret.xml.j2"
+    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.xml"
+    mode: "0600"
+  when: inventory_hostname in groups['compute']
+
+- name: Extract key from file
+  local_action: shell cat {{ cephx_keyring_files.files[0].path }} | grep -E 'key\s*=' | awk '{ print $3 }'
+  register: cephx_raw_key
+
+- name: Pushing secrets key for libvirt
+  copy:
+    content: "{{ cephx_raw_key.stdout }}"
+    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ rbd_secret_uuid }}.base64"
+    mode: "0600"
+  when: inventory_hostname in groups['compute']

ansible/roles/nova/templates/nova-compute.json.j2

@@ -6,18 +6,12 @@
             "dest": "/etc/nova/nova.conf",
             "owner": "nova",
             "perm": "0600"
-        }{% if enable_ceph | bool %},
+        }{% if nova_backend == "rbd" %},
         {
-            "source": "{{ container_config_directory }}/ceph.client.nova.keyring",
+            "source": "{{ container_config_directory }}/ceph.*",
-            "dest": "/etc/ceph/ceph.client.nova.keyring",
+            "dest": "/etc/ceph/",
             "owner": "nova",
-            "perm": "0600"
+            "perm": "0700"
-        },
-        {
-            "source": "{{ container_config_directory }}/ceph.conf",
-            "dest": "/etc/ceph/ceph.conf",
-            "owner": "nova",
-            "perm": "0600"
         }{% endif %}
     ]
 }

ansible/roles/nova/templates/nova-libvirt.json.j2

@@ -12,7 +12,7 @@
             "dest": "/etc/libvirt/qemu.conf",
             "owner": "root",
             "perm": "0644"
-        }{% if enable_ceph | bool %},
+        }{% if nova_backend == "rbd" %},
         {
             "source": "{{ container_config_directory }}/secrets",
             "dest": "/etc/libvirt/secrets",

ansible/roles/nova/templates/nova.conf.j2

@@ -153,15 +153,19 @@ memcached_servers = {% for host in groups['memcached'] %}{{ hostvars[host]['ansi
 
 [libvirt]
 connection_uri = "qemu+tcp://{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}/system"
-{% if enable_ceph | bool %}
+{% if enable_ceph | bool and nova_backend == "rbd" %}
 images_type = rbd
 images_rbd_pool = {{ ceph_nova_pool_name }}
 images_rbd_ceph_conf = /etc/ceph/ceph.conf
 rbd_user = nova
-rbd_secret_uuid = {{ rbd_secret_uuid }}
 disk_cachemodes="network=writeback"
 hw_disk_discard = unmap
 {% endif %}
+{% if nova_backend == "rbd" %}
+rbd_secret_uuid = {{ rbd_secret_uuid }}
+{% endif %}
+
+
 
 [upgrade_levels]
 compute = auto

doc/external-ceph-guide.rst

@@ -14,7 +14,7 @@ Requirements
 * An existing installation of Ceph
 * Existing Ceph storage pools
 * Existing credentials in Ceph for OpenStack services to connect to Ceph
-(Glance, Cinder)
+(Glance, Cinder, Nova)
 
 Enabling External Ceph
 ======================
@@ -62,11 +62,9 @@ Step 1 is done by using Kolla's INI merge mechanism: Create a file in
   [glance_store]
   stores = rbd
   default_store = rbd
-  rbd_store_chunk_size = 8
   rbd_store_pool = images
   rbd_store_user = glance
   rbd_store_ceph_conf = /etc/ceph/ceph.conf
-  rados_connect_timeout = 0
 
   [image_format]
   container_formats = bare
@@ -151,3 +149,31 @@ cinder-volume and cinder-backup directories:
           key = AQAg5YRXpChaGRAAlTSCleesthCRmCYrfQVX1w==
 
 It is important that the files are named ceph.client*.
+
+Nova
+------
+
+In ``/etc/kolla/global.yml`` set
+
+::
+
+  nova_backend_ceph: "yes"
+
+Put ceph.conf and keyring file into ``/etc/kolla/config/nova``:
+
+::
+
+  $ ls /etc/kolla/config/nova
+  ceph.client.nova.keyring  ceph.conf
+
+Configure nova-compute to use Ceph as the ephemeral backend by creating ``/etc/kolla/config/nova/nova-compute.conf`` and adding the following contents:
+
+::
+
+  [libvirt]
+  images_rbd_pool=vms
+  images_type=rbd
+  images_rbd_ceph_conf=/etc/ceph/ceph.conf
+  rbd_user=nova
+
+NOTE: rbd_user might vary depending on your environment.

etc/kolla/globals.yml

@@ -157,6 +157,12 @@ cinder_backend_ceph: "{{ enable_ceph }}"
 #cinder_volume_group:
 
 
+#######################
+# Nova options
+#######################
+nova_backend_ceph: "{{ enable_ceph }}"
+
+
 #######################################
 # Manila - Shared File Systems Options
 #######################################