Add Ceph support for Mariadb

Prior code left room for ceph support, but did not implement it.  This
patch finishes the job, but only for Mariadb.

Change-Id: I73af214dbee770ea496c9636a6b947567ff80b0c
Partially-implements: blueprint kolla-kubernetes-cli

doc/source/multi-node.rst

@@ -168,6 +168,7 @@ Operator Create Resources
 
 ::
 
+  kolla-kubernetes bootstrap ceph  # adds ceph secret, no-op for storage_provider!=ceph
   kolla-kubernetes bootstrap mariadb
   sleep 30  # wait for mariadb bootstrap to finish
   kolla-kubernetes resource delete mariadb bootstrap  # workaround known issue #1
@@ -254,6 +255,7 @@ Operator Delete Resources
   kolla-kubernetes kill keystone
   kolla-kubernetes kill memcached
   kolla-kubernetes kill mariadb
+  kolla-kubernetes kill ceph
 
 
 Workflow Engine Delete Resources

etc/kolla-kubernetes/kolla-kubernetes.yml

@@ -12,13 +12,13 @@
 ########################
 # Kubernetes Cluster
 ########################
-keystone_replicas: "1"
-memcached_replicas: "1"
-rabbitmq_replicas: "1"
-horizon_replicas: "1"
-glance_api_replicas: "1"
-glance_registry_replicas: "1"
-neutron_control_replicas: "1"
+keystone_replicas: "1"  # may be > 1
+memcached_replicas: "1"  # must == 1 even for multinode
+rabbitmq_replicas: "1"  # must == 1 even for multinode
+horizon_replicas: "1"  # may be > 1
+glance_api_replicas: "1"  # may be > 1
+glance_registry_replicas: "1"  # must == 1 even for multinode
+neutron_control_replicas: "1"  # may be > 1
 nova_api_replicas: "1"
 nova_conductor_replicas: "1"
 nova_scheduler_replicas: "1"
@@ -35,14 +35,53 @@ dns_domain_name: "openstack.kolla"
 ########################
 storage_provider: "host"  # host, ceph, gce, aws
 storage_ceph:
-  keyring: /etc/ceph/ceph.client.admin.keyring
+  # - WARNING: These sample defaults configure ceph access using the
+  #   ceph "admin" user/key, because it involves the least amount of
+  #   work for the a user to get ceph volumes working.  However, it is
+  #   highly recommended that the operator create a dedicated ceph
+  #   user/key with access only to the ceph pool to be used by this
+  #   Kubernetes cluster.
+  #
+  # Kubernetes nodes act as ceph clients because they must mount ceph
+  #   volumes on behalf of pods.  For a particular ceph user, there
+  #   are two ways to pass the ceph secret keyring to Kubernetes.
+  #
+  #   1) The ceph user secret keyring may be loaded as a kubernetes
+  #      secret.  The base64-encoded secret must be referenced by
+  #      storage_ceph.key.  To disable this method, comment out the
+  #      storage_ceph.key definition or set the value to "".  The
+  #      encoded secret may be created with this command:
+  #      $ ssh ceph-mon cat /etc/ceph/path/to/ceph.client.keyring \
+  #          | grep key | awk '{print $3}' | base64
+  #
+  #   2) The ceph user secret keyring may be stored on the Kubernetes
+  #      node's filesystem, and then referenced by
+  #      storage_ceph.keyring.  To disable this method, comment out
+  #      the storage_ceph.keyring definition or set the value to "".
+  #
+  # If both configurations are defined, Method 1) above takes
+  #   precedence over method 2).  Prefer using Method 1) to avoid
+  #   provisioning the ceph key on every node, which is difficult if
+  #   using cloud provider auto-provisioning.
+  #
+  # List of ceph monitor nodes
   monitors:
   - x.x.x.x
   - y.y.y.y
-  pool: rbd
-  secretName: pkt-ceph-secret
-  ssh_user: root
+  # Default ceph user for authenticated access
   user: admin
+  # The default pool to locate ceph volumes
+  pool: rbd
+  # Default user to use in order to run remote SSH commands
+  #   e.g. kolla-kubernetes may execute:
+  #        ssh root@ceph-mon rbd create pool/resourceName --size 1024
+  ssh_user: root
+  # Any unique secret string within the kube env
+  secretName: ceph-secret
+  # The base64-encoded secret key which nodes need for mounting ceph volumes
+  key: EXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLE=
+  # The ceph keyring file location on each kubernetes node's filesystem
+  keyring: /etc/ceph/ceph.client.admin.keyring
 
 ################################
 # Persistent volumes sizes in GB

etc/kolla-kubernetes/service_resources.yml

@@ -20,15 +20,26 @@
 #   jinja references, and are passed unmodified directly to the
 #   template.
 
-
 kolla-kubernetes:
   services:
+  - name: ceph
+    pods:
+    resources:
+      secret:
+      - services/ceph/ceph-secret.yml.j2
+      disk:
+      pv:
+      pvc:
+      svc:
+      bootstrap:
+      pod:
   - name: mariadb
     pods:
     - name: mariadb
       containers:
       - name: mariadb
     resources:
+      secret:
       disk:
       - name: mariadb-disk
         template: services/common/common-disk.sh.j2
@@ -59,6 +70,7 @@ kolla-kubernetes:
       containers:
       - name: memcached
     resources:
+      secret:
       disk:
       pv:
       pvc:
@@ -75,6 +87,7 @@ kolla-kubernetes:
       containers:
       - name: keystone
     resources:
+      secret:
       disk:
       pv:
       pvc:
@@ -95,6 +108,7 @@ kolla-kubernetes:
       containers:
       - name: horizon
     resources:
+      secret:
       disk:
       pv:
       pvc:
@@ -111,6 +125,7 @@ kolla-kubernetes:
       containers:
       - name: rabbitmq
     resources:
+      secret:
       disk:
       pv:
       pvc:
@@ -132,6 +147,7 @@ kolla-kubernetes:
       - name: glance-api
       - name: glance-registry
     resources:
+      secret:
       disk:
       - name: glance-disk
         template: services/common/common-disk.sh.j2
@@ -172,6 +188,7 @@ kolla-kubernetes:
       - name: nova-scheduler
       - name: nova-conductor
     resources:
+      secret:
       disk:
       pv:
       pvc:
@@ -211,6 +228,7 @@ kolla-kubernetes:
       containers:
       - name: neutron-server
     resources:
+      secret:
       disk:
       pv:
       pvc:
@@ -259,6 +277,7 @@ kolla-kubernetes:
       containers:
       - name: swift-proxy-server
     resources:
+      secret:
       disk:
       pv:
       pvc:
@@ -292,6 +311,7 @@ kolla-kubernetes:
       - name: skydns
       - name: healthz
     resources:
+      secret:
       disk:
       pv:
       pvc:

kolla_kubernetes/service_resources.py

@@ -118,14 +118,16 @@ class KollaKubernetesResources(object):
 
 class Service(object):
     VALID_ACTIONS = 'create delete status'.split(" ")
-    VALID_RESOURCE_TYPES = 'configmap disk pv pvc svc bootstrap pod'.split(" ")
+    VALID_RESOURCE_TYPES = ('configmap secret '
+                            'disk pv pvc svc bootstrap pod').split(" ")
     # Keep old logic for LEGACY support of bootstrap, run, and kill commands
     #   Legacy commands did not keep order.  Here, we define order.
     #   Hoping to get rid of the LEGACY commands entirely if people okay.
     #   Otherwise, we wait until Ansible workflow engine.
     #   SVC should really be in bootstrap command, since it is stateful
     #   CONFIGMAP remains listed twice, since that was the old logic.
-    LEGACY_BOOTSTRAP_RESOURCES = 'configmap disk pv pvc bootstrap'.split(" ")
+    LEGACY_BOOTSTRAP_RESOURCES = ('configmap secret '
+                                  'disk pv pvc bootstrap').split(" ")
     LEGACY_RUN_RESOURCES = 'configmap svc pod'.split(" ")
 
     def __init__(self, y):

services/ceph/ceph-secret.yml.j2

@@ -0,0 +1,23 @@
+{%- set resourceName = kolla_kubernetes.cli.args.service_name %}
+
+{%- if storage_provider != "ceph" %}
+  # NO-OP for Non-Ceph storage providers
+
+{%- else %}
+
+  {%- if storage_ceph.secretName is defined and storage_ceph.secretName|length>0 and storage_ceph.key is defined and storage_ceph.key|length>0 %}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ storage_ceph.secretName }}
+data:
+  {#- Create base64-encoded key:
+      ssh ceph-mon cat /etc/ceph/path/to/ceph.client.keyring | grep key | awk '{print $3}' | base64 #}
+  key: {{ storage_ceph.key }}
+
+  {%- else %}
+  # NO-OP since either storage_ceph.secretName or storage_ceph.key are undefined
+
+  {%- endif %}
+
+{%- endif %}

services/common/common-disk.sh.j2

@@ -9,11 +9,12 @@
   {%- else %}
     {{ raise('Unknown action') }}
   {%- endif %}
+
 {%- elif storage_provider == "ceph" %}
   {%- if kolla_kubernetes.cli.args.action == "create" %}
   ssh {{ storage_ceph.ssh_user -}} @ {{- storage_ceph.monitors[0] }} rbd create {{ storage_ceph.pool -}}/{{- resourceName }} --size "{{ kolla_kubernetes.template.vars.size_in_gb | int * 1024 }}" --image-feature layering
   {%- elif kolla_kubernetes.cli.args.action == "delete" %}
-  ssh {{ storage_ceph.ssh_user -}} @ {{- storage_ceph.monitors[0] }} rbd delete {{ storage_ceph.pool -}}/{{- resourceName }}
+  ssh {{ storage_ceph.ssh_user -}} @ {{- storage_ceph.monitors[0] }} rbd rm {{ storage_ceph.pool -}}/{{- resourceName }}
   {%- else %}
     {{ raise('Unknown action') }}
   {%- endif %}

services/common/common-pv.yml.j2

@@ -29,9 +29,13 @@ spec:
     pool: {{ storage_ceph.pool }}
     image: {{ resourceName }}
     user: {{ storage_ceph.user }}
+    {%- if storage_ceph.keyring is defined and storage_ceph.keyring|length>0 %}
     keyring: {{ storage_ceph.keyring }}
+    {%- endif %}
+    {%- if storage_ceph.secretName is defined and storage_ceph.secretName|length>0 and storage_ceph.key is defined and storage_ceph.key|length>0 %}
     secretRef:
       name: {{ storage_ceph.secretName }}
+    {%- endif %}
     fsType: ext4
     readOnly: false
 {%- endif %}