Ceph support

Example usage:
configs:
  ceph:
    fsid: "afca8524-2c47-4b81-a0b7-2300e62212f9"
    mon_host: "IP OF CEPH MON NODE"
  cinder:
    ceph:
      enable: true
      key: "AQBShfJXID9pFRAAm4VLpbNXa4XJ9zgAh7dm2g=="
      rbd_secret_uuid: "b416770d-f3d4-4ac9-b6db-b6a7ac1c61c0"
  nova:
    ceph:
      enable: true

Change-Id: I2be9526f55739a5743709e10b5ce84c2cee601a9
Depends-On: Iefcebfe55544331b379f4ee1c4001ec33f539a28

docker/nova-base/Dockerfile.j2

@@ -1,7 +1,10 @@
 FROM {{ image_spec("openstack-base") }}
 MAINTAINER {{ maintainer }}
 
-RUN apt-get install -y --no-install-recommends \
+COPY sources.list.debian /etc/apt/sources.list.d/ceph.list
+RUN apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 460F3994 \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
         bridge-utils \
         openvswitch-switch \
         iptables \
@@ -22,6 +25,5 @@ RUN useradd --user-group --create-home --home-dir /var/lib/nova nova \
 
 COPY nova_sudoers /etc/sudoers.d/nova_sudoers
 RUN chmod 750 /etc/sudoers.d \
-    && chmod 440 /etc/sudoers.d/nova_sudoers
-
-RUN usermod -a -G microservices nova
+    && chmod 440 /etc/sudoers.d/nova_sudoers \
+    && usermod -a -G microservices nova

docker/nova-base/sources.list.debian

@@ -0,0 +1 @@
+deb http://download.ceph.com/debian-jewel jessie main

docker/nova-compute/Dockerfile.j2

@@ -6,10 +6,17 @@ RUN apt-get -y install --no-install-recommends \
         python-libvirt \
         qemu-utils \
         ceph-common \
+        python-ceph \
+        python-rados \
         genisoimage \
         open-iscsi \
     && apt-get clean \
-    && mkdir -p /etc/ceph
+    && mkdir -p /etc/ceph \
+    && chown -R nova: /etc/ceph \
+    && ln -s /usr/lib/python2.7/dist-packages/rados.so /var/lib/microservices/venv/local/lib/python2.7/site-packages/rados.so \
+    && ln -s /usr/lib/python2.7/dist-packages/rados-0.egg-info /var/lib/microservices/venv/local/lib/python2.7/site-packages/rados-0.egg-info \
+    && ln -s /usr/lib/python2.7/dist-packages/rbd-0.egg-info /var/lib/microservices/venv/local/lib/python2.7/site-packages/rbd-0.egg-info \
+    && ln -s /usr/lib/python2.7/dist-packages/rbd.so /var/lib/microservices/venv/local/lib/python2.7/site-packages/rbd.so
 
 RUN /var/lib/microservices/venv/bin/pip install --upgrade libvirt-python rtslib-fb && \
     rm -rf requirements && \

docker/nova-libvirt/Dockerfile.j2

@@ -3,6 +3,8 @@ MAINTAINER {{ maintainer }}
 
 RUN apt-get -y install --no-install-recommends \
         ceph-common \
+        python-ceph \
+        python-rados \
         libvirt-bin \
         dmidecode \
         pm-utils \
@@ -13,4 +15,8 @@ RUN apt-get -y install --no-install-recommends \
     && apt-get clean \
     && mkdir -p /etc/ceph \
     && rm -f /etc/libvirt/qemu/networks/default.xml /etc/libvirt/qemu/networks/autostart/default.xml \
-    && usermod -a -G libvirt nova
+    && usermod -a -G libvirt nova \
+    && ln -s /usr/lib/python2.7/dist-packages/rados.so /var/lib/microservices/venv/local/lib/python2.7/site-packages/rados.so \
+    && ln -s /usr/lib/python2.7/dist-packages/rados-0.egg-info /var/lib/microservices/venv/local/lib/python2.7/site-packages/rados-0.egg-info \
+    && ln -s /usr/lib/python2.7/dist-packages/rbd-0.egg-info /var/lib/microservices/venv/local/lib/python2.7/site-packages/rbd-0.egg-info \
+    && ln -s /usr/lib/python2.7/dist-packages/rbd.so /var/lib/microservices/venv/local/lib/python2.7/site-packages/rbd.so

service/files/ceph-libvirt-secret-config.sh.j2

@@ -0,0 +1,4 @@
+# This whole approach is very insecure. Fixme.
+virsh secret-define --file /etc/libvirt/secrets/secret.xml
+virsh secret-set-value --secret {{ cinder.ceph.rbd_secret_uuid }} --base64 {{ cinder.ceph.key }}
+rm -f /etc/libvirt/secrets/secret.xml /opt/ccp/bin/ceph-libvirt-secret-config.sh

service/files/ceph.client.cinder.keyring.j2

@@ -0,0 +1,3 @@
+[client.cinder]
+key = {{ cinder.ceph.key }}
+

service/files/ceph.conf.j2

@@ -0,0 +1,7 @@
+[global]
+fsid = {{ ceph.fsid }}
+mon_host = {{ ceph.mon_host }}
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+

service/files/defaults.yaml

@@ -15,6 +15,10 @@ configs:
     console: "novnc"
     virt_type: "kvm"
 
+    ceph:
+      enable: false
+      pool_name: "vms"
+
 sources:
   openstack/nova:
     git_url: https://github.com/openstack/nova.git

service/files/nova.conf.j2

@@ -28,7 +28,7 @@ linuxnet_interface_driver = nova.network.linux_net.LinuxOVSInterfaceDriver
 linuxnet_interface_driver = nova.network.linux_net.BridgeInterfaceDriver
 {% endif %}
 
-allow_resize_to_same_host = true
+allow_resize_to_same_host = True
 
 compute_driver = libvirt.LibvirtDriver
 
@@ -56,7 +56,7 @@ novncproxy_base_url = http://{{ address('nova-novncproxy') }}:{{ nova.novncproxy
 {% elif nova.console == 'spice' %}
 [vnc]
 # We have to turn off vnc to use spice
-enabled = false
+enabled = False
 [spice]
 server_listen = {{ network_topology["private"]["address"] }}
 server_proxyclient_address = {{ nova.spicehtml5proxy.host }}
@@ -97,7 +97,7 @@ api_servers = {{ address('glance-api') }}:{{ glance.api_port }}
 num_retries = 1
 
 [cinder]
-catalog_info = volume:cinder:internalURL
+catalog_info = volumev2:cinder:internalURL
 
 [neutron]
 url = http://{{ address('neutron-server') }}:{{ neutron.server_port }}
@@ -145,12 +145,12 @@ memcached_servers = {{ address('memcached') }}:{{ memcached.port }}
 virt_type = {{ nova.virt_type }}
 #Need to split file in sections to avoid confussion with wrong IP here for non compute services
 connection_uri = "qemu+tcp://{{ network_topology["private"]["address"] }}/system"
-{% if nova.enable_ceph %}
+{% if nova.ceph.enable %}
 images_type = rbd
-images_rbd_pool = {{ ceph_nova_pool_name }}
+images_rbd_pool = {{ nova.ceph.pool_name }}
 images_rbd_ceph_conf = /etc/ceph/ceph.conf
-rbd_user = nova
-rbd_secret_uuid = {{ rbd_secret_uuid }}
+rbd_user = {{ cinder.ceph.username }}
+rbd_secret_uuid = {{ cinder.ceph.rbd_secret_uuid }}
 disk_cachemodes="network=writeback"
 live_migration_flag="VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST,VIR_MIGRATE_TUNNELLED"
 hw_disk_discard = unmap

service/files/rbd_secret.xml.j2

@@ -0,0 +1,6 @@
+<secret ephemeral='no' private='no'>
+  <uuid>{{ cinder.ceph.rbd_secret_uuid }}</uuid>
+  <usage type='ceph'>
+    <name>client.cinder secret</name>
+  </usage>
+</secret>

service/nova-compute.yaml

@@ -36,8 +36,19 @@ service:
           - nova-libvirt:local
         files:
           - nova.conf
+          # {% if nova.ceph.enable %}
+          - ceph-conf
+          - nova-ceph-key
+          # {% endif %}
+
 files:
   nova.conf:
     path: /etc/nova/nova.conf
     content: nova.conf.j2
     perm: "0600"
+  ceph-conf:
+    path: /etc/ceph/ceph.conf
+    content: ceph.conf.j2
+  nova-ceph-key:
+    path: /etc/ceph/ceph.client.cinder.keyring
+    content: ceph.client.cinder.keyring.j2

service/nova-libvirt.yaml

@@ -33,6 +33,16 @@ service:
           - libvirtd.conf
           - nova-libvirt-bootstrap.sh
           - qemu.conf
+          # {% if nova.ceph.enable %}
+          - rbd-secret
+          - ceph-conf
+          - nova-ceph-key
+          - ceph-libvirt-secret-config.sh
+      post:
+        - name: nova-libvirt-secret-set
+          command: /opt/ccp/bin/ceph-libvirt-secret-config.sh
+          # {% endif %}
+
 files:
   libvirtd.conf:
     path: /etc/libvirt/libvirtd.conf
@@ -46,3 +56,17 @@ files:
     path: /etc/libvirt/qemu.conf
     content: qemu.conf.j2
     perm: "0600"
+  rbd-secret:
+    path: /etc/libvirt/secrets/secret.xml
+    content: rbd_secret.xml.j2
+    perm: "0600"
+  ceph-conf:
+    path: /etc/ceph/ceph.conf
+    content: ceph.conf.j2
+  nova-ceph-key:
+    path: /etc/ceph/ceph.client.cinder.keyring
+    content: ceph.client.cinder.keyring.j2
+  ceph-libvirt-secret-config.sh:
+    path: /opt/ccp/bin/ceph-libvirt-secret-config.sh
+    content: ceph-libvirt-secret-config.sh.j2
+    perm: "0755"