CephFS: support for OSH

This PS adds support for CephFS in OSH. Change-Id: I54d5302b89f04cc30f151c8db59f476f02be5652
2018-01-08 17:45:33 -05:00 · 2018-01-08 17:45:33 -05:00 · dc105fdcfa
parent a78e638897
commit dc105fdcfa
22 changed files with 519 additions and 49 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -51,6 +51,8 @@
    name: openstack-helm-dev-deploy
    vars:
        zuul_osh_infra_relative_path: ../openstack-helm-infra/
+    pre-run:
+      - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-upgrade-host.yaml
    run: tools/gate/playbooks/dev-deploy.yaml
    post-run: ../openstack-helm-infra/tools/gate/playbooks/osh-infra-collect-logs.yaml
    required-projects:
@ -63,6 +65,7 @@
        zuul_osh_infra_relative_path: ../openstack-helm-infra/
    name: openstack-helm-legacy
    pre-run:
+      - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-upgrade-host.yaml
      - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-deploy-docker.yaml
      - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-build.yaml
      - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-deploy-k8s.yaml
--- a/ceph/templates/bin/_ceph-cephfs-client-key.sh.tpl
+++ b/ceph/templates/bin/_ceph-cephfs-client-key.sh.tpl
@ -0,0 +1,43 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+CEPH_CEPHFS_KEY=$(kubectl get secret ${PVC_CEPH_CEPHFS_STORAGECLASS_ADMIN_SECRET_NAME} \
+    --namespace=${PVC_CEPH_CEPHFS_STORAGECLASS_DEPLOYED_NAMESPACE} \
+    -o json )
+
+ceph_activate_namespace() {
+  kube_namespace=$1
+  secret_type=$2
+  secret_name=$3
+  ceph_key=$4
+  {
+  cat <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "${secret_name}"
+type: "${secret_type}"
+data:
+  key: $( echo ${ceph_key} )
+EOF
+  } | kubectl create --namespace ${kube_namespace} -f -
+}
+
+ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/cephfs" ${PVC_CEPH_CEPHFS_STORAGECLASS_USER_SECRET_NAME} "$(echo ${CEPH_CEPHFS_KEY} | jq -r '.data | .[]')"
--- a/ceph/templates/bin/_ceph-namespace-client-key-cleaner.sh.tpl
+++ b/ceph/templates/bin/_ceph-namespace-client-key-cleaner.sh.tpl
@ -21,4 +21,4 @@ set -ex
 kubectl delete secret \
  --namespace ${DEPLOYMENT_NAMESPACE} \
  --ignore-not-found=true \
-  ${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME}
+  ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME}
--- a/ceph/templates/bin/_ceph-namespace-client-key.sh.tpl
+++ b/ceph/templates/bin/_ceph-namespace-client-key.sh.tpl
@ -18,22 +18,26 @@ limitations under the License.

 set -ex

+CEPH_RBD_KEY=$(kubectl get secret ${PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME} \
+    --namespace=${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} \
+    -o json )
+
 ceph_activate_namespace() {
  kube_namespace=$1
-  CEPH_KEY=$(kubectl get secret ${PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME} \
-      --namespace=${PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE} \
-      -o json | jq -r '.data | .[]')
+  secret_type=$2
+  secret_name=$3
+  ceph_key=$4
  {
  cat <<EOF
 apiVersion: v1
 kind: Secret
 metadata:
-  name: "${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME}"
-type: kubernetes.io/rbd
+  name: "${secret_name}"
+type: "${secret_type}"
 data:
-  key: $(echo ${CEPH_KEY})
+  key: $( echo ${ceph_key} )
 EOF
  } | kubectl create --namespace ${kube_namespace} -f -
 }

-ceph_activate_namespace ${DEPLOYMENT_NAMESPACE}
+ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME} "$(echo ${CEPH_RBD_KEY} | jq -r '.data | .[]')"
--- a/ceph/templates/bin/_cephfs-provisioner.sh.tpl
+++ b/ceph/templates/bin/_cephfs-provisioner.sh.tpl
@ -0,0 +1,21 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+exec /usr/local/bin/cephfs-provisioner -id ${POD_NAME}
--- a/ceph/templates/configmap-bin.yaml
+++ b/ceph/templates/configmap-bin.yaml
@ -73,6 +73,10 @@ data:
 {{ tuple "bin/_check_zombie_mons.py.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
  rbd-provisioner.sh: |
 {{ tuple "bin/_rbd-provisioner.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+  cephfs-provisioner.sh: |
+{{ tuple "bin/_cephfs-provisioner.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+  ceph-cephfs-client-key.sh: |
+{{ tuple "bin/_ceph-cephfs-client-key.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
  ceph-osd-liveness-readiness.sh: |
 {{ tuple "bin/_ceph-osd-liveness-readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
  ceph-mon-liveness.sh: |
--- a/ceph/templates/deployment-cephfs-provisioner.yaml
+++ b/ceph/templates/deployment-cephfs-provisioner.yaml
@ -0,0 +1,175 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.deployment_cephfs_provisioner }}
+{{- $envAll := . }}
+{{- if .Values.deployment.cephfs_provisioner }}
+{{- $dependencies := .Values.dependencies.cephfs_provisioner }}
+
+{{- $serviceAccountName := "ceph-cephfs-provisioner"}}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - ''
+    resources:
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - ''
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - events
+    verbs:
+      - list
+      - watch
+      - create
+      - update
+      - patch
+  - apiGroups:
+      - ''
+    resources:
+      - services
+      - endpoints
+    verbs:
+      - get
+  - apiGroups:
+      - extensions
+    resources:
+      - podsecuritypolicies
+    resourceNames:
+      - cephfs-provisioner
+    verbs:
+      - use
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: run-cephfs-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  name: ceph-cephfs-provisioner
+spec:
+  replicas: {{ .Values.pod.replicas.cephfs_provisioner }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "cephfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      affinity:
+{{ tuple $envAll "cephfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+      initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: ceph-cephfs-provisioner
+          image: {{ .Values.images.tags.ceph_cephfs_provisioner }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.cephfs_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: PROVISIONER_NAME
+              value: {{ .Values.storageclass.rbd.provisioner }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          command:
+            - /tmp/cephfs-provisioner.sh
+          volumeMounts:
+            - name: ceph-bin
+              mountPath: /tmp/cephfs-provisioner.sh
+              subPath: cephfs-provisioner.sh
+              readOnly: true
+      volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
+{{- end }}
+{{- end }}
--- a/ceph/templates/deployment-rbd-provisioner.yaml
+++ b/ceph/templates/deployment-rbd-provisioner.yaml
@ -144,7 +144,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
          env:
            - name: PROVISIONER_NAME
-              value: {{ .Values.storageclass.provisioner }}
+              value: {{ .Values.storageclass.rbd.provisioner }}
            - name: POD_NAME
              valueFrom:
                fieldRef:
--- a/ceph/templates/job-cephfs-client-key.yaml
+++ b/ceph/templates/job-cephfs-client-key.yaml
@ -0,0 +1,127 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_cephfs_client_key }}
+{{- $envAll := . }}
+{{- if .Values.deployment.cephfs_provisioner }}
+{{- $dependencies := .Values.dependencies.cephfs_client_key_generator }}
+
+{{- $randStringSuffix := randAlphaNum 5 | lower }}
+
+{{- $serviceAccountName := "ceph-cephfs-client-key-generator" }}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+  namespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+  namespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-cephfs-client-key-generator
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name:  ceph-storage-keys-generator
+          image: {{ .Values.images.tags.ceph_config_helper }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: PVC_CEPH_CEPHFS_STORAGECLASS_USER_SECRET_NAME
+              value: {{ .Values.storageclass.cephfs.user_secret_name }}
+            - name: PVC_CEPH_CEPHFS_STORAGECLASS_ADMIN_SECRET_NAME
+              value: {{ .Values.storageclass.cephfs.admin_secret_name }}
+            - name: PVC_CEPH_CEPHFS_STORAGECLASS_DEPLOYED_NAMESPACE
+              value: {{ .Values.storageclass.cephfs.admin_secret_namespace }}
+          command:
+            - /opt/ceph/ceph-cephfs-client-key.sh
+          volumeMounts:
+            - name: ceph-bin
+              mountPath: /opt/ceph/ceph-cephfs-client-key.sh
+              subPath: ceph-cephfs-client-key.sh
+              readOnly: true
+      volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
+{{- end }}
+{{- end }}
--- a/ceph/templates/job-namespace-client-key-cleaner.yaml
+++ b/ceph/templates/job-namespace-client-key-cleaner.yaml
@ -83,8 +83,8 @@ spec:
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
-            - name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME
-              value: {{ .Values.storageclass.user_secret_name }}
+            - name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
+              value: {{ .Values.storageclass.rbd.user_secret_name }}
          command:
            - /opt/ceph/ceph-namespace-client-key-cleaner.sh
          volumeMounts:
--- a/ceph/templates/job-namespace-client-key.yaml
+++ b/ceph/templates/job-namespace-client-key.yaml
@ -55,7 +55,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: Role
 metadata:
  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
-  namespace: {{ .Values.storageclass.admin_secret_namespace }}
+  namespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
 rules:
  - apiGroups:
      - ""
@ -69,7 +69,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: RoleBinding
 metadata:
  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
-  namespace: {{ .Values.storageclass.admin_secret_namespace }}
+  namespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
@ -105,12 +105,12 @@ spec:
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
-            - name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME
-              value: {{ .Values.storageclass.user_secret_name }}
-            - name: PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME
-              value: {{ .Values.storageclass.admin_secret_name }}
-            - name: PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE
-              value: {{ .Values.storageclass.admin_secret_namespace }}
+            - name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
+              value: {{ .Values.storageclass.rbd.user_secret_name }}
+            - name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME
+              value: {{ .Values.storageclass.rbd.admin_secret_name }}
+            - name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
+              value: {{ .Values.storageclass.rbd.admin_secret_namespace }}
          command:
            - /opt/ceph/ceph-namespace-client-key.sh
          volumeMounts:
--- a/ceph/templates/job-storage-admin-keys.yaml
+++ b/ceph/templates/job-storage-admin-keys.yaml
@ -85,7 +85,7 @@ spec:
            - name: CEPH_KEYRING_ADMIN_NAME
              value: {{ .Values.secrets.keyrings.admin }}
            - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME
-              value: {{ .Values.storageclass.admin_secret_name }}
+              value: {{ .Values.storageclass.rbd.admin_secret_name }}
          command:
            - /opt/ceph/ceph-storage-key.sh
          volumeMounts:
--- a/ceph/templates/storageclass-cephfs.yaml
+++ b/ceph/templates/storageclass-cephfs.yaml
@ -0,0 +1,34 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.storageclass }}
+{{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
+{{- if .Values.storageclass.cephfs.provision_storage_class }}
+---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: {{ .Values.storageclass.cephfs.name }}
+provisioner: {{ .Values.storageclass.cephfs.provisioner }}
+parameters:
+    monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+    adminId: {{ .Values.storageclass.cephfs.admin_id | quote }}
+    adminSecretName: {{ .Values.storageclass.cephfs.user_secret_name | quote }}
+    adminSecretNamespace: {{ .Release.Namespace | quote }}
+{{- end }}
+{{- end }}
+{{- end }}
--- a/ceph/templates/storageclass.yaml
+++ b/ceph/templates/storageclass.yaml
@ -17,23 +17,23 @@ limitations under the License.
 {{- if .Values.manifests.storageclass }}
 {{- $envAll := . }}
 {{- if .Values.deployment.ceph }}
-{{- if .Values.storageclass.provision_storage_class }}
+{{- if .Values.storageclass.rbd.provision_storage_class }}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-    name: {{ .Values.storageclass.name }}
-provisioner: {{ .Values.storageclass.provisioner }}
+    name: {{ .Values.storageclass.rbd.name }}
+provisioner: {{ .Values.storageclass.rbd.provisioner }}
 parameters:
    monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-    adminId: {{ .Values.storageclass.admin_id }}
-    adminSecretName: {{ .Values.storageclass.admin_secret_name }}
-    adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace }}
-    pool: {{ .Values.storageclass.pool }}
-    userId: {{ .Values.storageclass.user_id }}
-    userSecretName: {{ .Values.storageclass.user_secret_name }}
-    imageFormat: {{ .Values.storageclass.image_format | quote }}
-    imageFeatures: {{ .Values.storageclass.image_features }}
+    adminId: {{ .Values.storageclass.rbd.admin_id }}
+    adminSecretName: {{ .Values.storageclass.rbd.admin_secret_name }}
+    adminSecretNamespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
+    pool: {{ .Values.storageclass.rbd.pool }}
+    userId: {{ .Values.storageclass.rbd.user_id }}
+    userSecretName: {{ .Values.storageclass.rbd.user_secret_name }}
+    imageFormat: {{ .Values.storageclass.rbd.image_format | quote }}
+    imageFeatures: {{ .Values.storageclass.rbd.image_features }}
 {{- end }}
 {{- end }}
 {{- end }}
--- a/ceph/values.yaml
+++ b/ceph/values.yaml
@ -17,6 +17,7 @@ deployment:
  storage_secrets: true
  client_secrets: true
  rbd_provisioner: true
+  cephfs_provisioner: true
  rgw_keystone_user_and_endpoints: false

 images:
@ -29,6 +30,7 @@ images:
    ceph_daemon: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04
    ceph_config_helper: docker.io/port/ceph-config-helper:v1.7.5
    ceph_rbd_provisioner: quay.io/external_storage/rbd-provisioner:v0.1.1
+    ceph_cephfs_provisioner: quay.io/external_storage/cephfs-provisioner:v0.1.1
  pull_policy: "IfNotPresent"

 labels:
@ -57,13 +59,14 @@ pod:
    rgw: 1
    mon_check: 1
    rbd_provisioner: 2
+    cephfs_provisioner: 2
    mgr: 1
  affinity:
-      anti:
-        type:
-          default: preferredDuringSchedulingIgnoredDuringExecution
-        topologyKey:
-          default: kubernetes.io/hostname
+    anti:
+      type:
+        default: preferredDuringSchedulingIgnoredDuringExecution
+      topologyKey:
+        default: kubernetes.io/hostname
  resources:
    enabled: false
    osd:
@ -108,6 +111,13 @@ pod:
      limits:
        memory: "50Mi"
        cpu: "500m"
+    cephfs_provisioner:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
    mgr:
      requests:
        memory: "5Mi"
@ -202,6 +212,8 @@ conf:
      mds:

 dependencies:
+  cephfs_client_key_generator:
+    jobs:
  job_keyring_generator:
    jobs:
  namespace_client_key_cleaner:
@ -252,6 +264,11 @@ dependencies:
    services:
    - service: ceph_mon
      endpoint: internal
+  cephfs_provisioner:
+    jobs:
+    services:
+    - service: ceph_mon
+      endpoint: internal
  ks_user:
    services:
    - service: identity
@ -303,18 +320,27 @@ bootstrap:
 # it is presumed you manage your own storage
 # class definition externally
 storageclass:
-  provision_storage_class: true
-  provisioner: ceph.com/rbd
-  name: general
-  monitors: null
-  pool: rbd
-  admin_id: admin
-  admin_secret_name: pvc-ceph-conf-combined-storageclass
-  admin_secret_namespace: ceph
-  user_id: admin
-  user_secret_name: pvc-ceph-client-key
-  image_format: "2"
-  image_features: layering
+  rbd:
+    provision_storage_class: true
+    provisioner: ceph.com/rbd
+    name: general
+    monitors: null
+    pool: rbd
+    admin_id: admin
+    admin_secret_name: pvc-ceph-conf-combined-storageclass
+    admin_secret_namespace: ceph
+    user_id: admin
+    user_secret_name: pvc-ceph-client-key
+    image_format: "2"
+    image_features: layering
+  cephfs:
+    provision_storage_class: true
+    provisioner: ceph.com/cephfs
+    name: cephfs
+    admin_id: admin
+    user_secret_name: pvc-ceph-cephfs-client-key
+    admin_secret_name: pvc-ceph-conf-combined-storageclass
+    admin_secret_namespace: ceph

 endpoints:
  cluster_domain_suffix: cluster.local
@ -388,9 +414,11 @@ manifests:
  deployment_mds: true
  deployment_moncheck: true
  deployment_rbd_provisioner: true
+  deployment_cephfs_provisioner: true
  deployment_rgw: true
  deployment_mgr: true
  job_bootstrap: true
+  job_cephfs_client_key: true
  job_keyring: true
  job_ks_endpoints: true
  job_ks_service: true
--- a/doc/source/install/developer/all-in-one.rst
+++ b/doc/source/install/developer/all-in-one.rst
@ -13,6 +13,10 @@ to package installation.
 Requirements
 ============

+.. warning:: Until the Ubuntu kernel shipped with 16.04 supports CephFS
+   subvolume mounts by default the `HWE Kernel
+   <../../troubleshooting/ubuntu-hwe-kernel.rst>`__ is required to use CephFS.
+
 System Requirements
 -------------------

--- a/doc/source/install/multinode.rst
+++ b/doc/source/install/multinode.rst
@ -28,6 +28,10 @@ documentation as we explore these options further.
 The installation procedures below, will take an administrator from a new
 ``kubeadm`` installation to Openstack-Helm deployment.

+.. warning:: Until the Ubuntu kernel shipped with 16.04 supports CephFS
+   subvolume mounts by default the `HWE Kernel
+   <../troubleshooting/ubuntu-hwe-kernel.rst>`__ is required to use CephFS.
+
 Kubernetes Preparation
 ======================

@ -581,4 +585,3 @@ The following shows a set of example hosts and the values fed into the configmap
        DEFAULT:
          vcpu_pin_set: "0-31"
          cpu_allocation_ratio: 3.0
-
--- a/doc/source/troubleshooting/index.rst
+++ b/doc/source/troubleshooting/index.rst
@ -9,6 +9,7 @@ Sometimes things go wrong. These guides will help you solve many common issues w

   database
   persistent-storage
+   ubuntu-hwe-kernel

 Getting help
 ============
--- a/doc/source/troubleshooting/ubuntu-hwe-kernel.rst
+++ b/doc/source/troubleshooting/ubuntu-hwe-kernel.rst
@ -0,0 +1,20 @@
+=================
+Ubuntu HWE Kernel
+=================
+
+To make use of CephFS in Ubuntu the HWE Kernel is required, until the issue
+described `here <https://github.com/kubernetes-incubator/external-storage/issues/345>`_
+is fixed.
+
+Installation
+============
+
+To deploy the HWE kernel, prior to deploying Kubernetes and OpenStack-Helm
+the following commands should be run on each node:
+
+.. code-block:: shell
+
+    #!/bin/bash
+    sudo -H apt-get update
+    sudo -H apt-get install -y linux-generic-hwe-16.04
+    sudo -H reboot now
--- a/tools/deployment/developer/04-ceph.sh
+++ b/tools/deployment/developer/04-ceph.sh
@ -31,6 +31,7 @@ helm install --namespace=ceph ${WORK_DIR}/ceph --name=ceph \
    --set deployment.storage_secrets=true \
    --set deployment.ceph=true \
    --set deployment.rbd_provisioner=true \
+    --set deployment.cephfs_provisioner=true \
    --set deployment.client_secrets=false \
    --set deployment.rgw_keystone_user_and_endpoints=false \
    --set bootstrap.enabled=true \
--- a/tools/deployment/developer/05-ceph-ns-activate.sh
+++ b/tools/deployment/developer/05-ceph-ns-activate.sh
@ -31,6 +31,7 @@ helm install --namespace=openstack ${WORK_DIR}/ceph --name=ceph-openstack-config
    --set deployment.storage_secrets=false \
    --set deployment.ceph=false \
    --set deployment.rbd_provisioner=false \
+    --set deployment.cephfs_provisioner=false \
    --set deployment.client_secrets=true \
    --set deployment.rgw_keystone_user_and_endpoints=false \
    --values=${WORK_DIR}/tools/overrides/mvp/ceph.yaml
--- a/tools/deployment/developer/10-ceph-radosgateway.sh
+++ b/tools/deployment/developer/10-ceph-radosgateway.sh
@ -31,6 +31,7 @@ helm install --namespace=openstack ${WORK_DIR}/ceph --name=radosgw-openstack \
   --set deployment.storage_secrets=false \
   --set deployment.ceph=false \
   --set deployment.rbd_provisioner=false \
+   --set deployment.cephfs_provisioner=false \
   --set deployment.client_secrets=false \
   --set deployment.rgw_keystone_user_and_endpoints=true \
   --values=${WORK_DIR}/tools/overrides/mvp/ceph.yaml