Merge "CephFS: support for OSH"

.zuul.yaml

@@ -51,6 +51,8 @@
     name: openstack-helm-dev-deploy
     vars:
         zuul_osh_infra_relative_path: ../openstack-helm-infra/
+    pre-run:
+      - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-upgrade-host.yaml
     run: tools/gate/playbooks/dev-deploy.yaml
     post-run: ../openstack-helm-infra/tools/gate/playbooks/osh-infra-collect-logs.yaml
     required-projects:
@@ -63,6 +65,7 @@
         zuul_osh_infra_relative_path: ../openstack-helm-infra/
     name: openstack-helm-legacy
     pre-run:
+      - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-upgrade-host.yaml
       - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-deploy-docker.yaml
       - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-build.yaml
       - ../openstack-helm-infra/tools/gate/playbooks/osh-infra-deploy-k8s.yaml

ceph/templates/bin/_ceph-cephfs-client-key.sh.tpl

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+CEPH_CEPHFS_KEY=$(kubectl get secret ${PVC_CEPH_CEPHFS_STORAGECLASS_ADMIN_SECRET_NAME} \
+    --namespace=${PVC_CEPH_CEPHFS_STORAGECLASS_DEPLOYED_NAMESPACE} \
+    -o json )
+
+ceph_activate_namespace() {
+  kube_namespace=$1
+  secret_type=$2
+  secret_name=$3
+  ceph_key=$4
+  {
+  cat <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "${secret_name}"
+type: "${secret_type}"
+data:
+  key: $( echo ${ceph_key} )
+EOF
+  } | kubectl create --namespace ${kube_namespace} -f -
+}
+
+ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/cephfs" ${PVC_CEPH_CEPHFS_STORAGECLASS_USER_SECRET_NAME} "$(echo ${CEPH_CEPHFS_KEY} | jq -r '.data | .[]')"

ceph/templates/bin/_ceph-namespace-client-key-cleaner.sh.tpl

@@ -21,4 +21,4 @@ set -ex
 kubectl delete secret \
   --namespace ${DEPLOYMENT_NAMESPACE} \
   --ignore-not-found=true \
-  ${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME}
+  ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME}

ceph/templates/bin/_ceph-namespace-client-key.sh.tpl

@@ -18,22 +18,26 @@ limitations under the License.
 
 set -ex
 
+CEPH_RBD_KEY=$(kubectl get secret ${PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME} \
+    --namespace=${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} \
+    -o json )
+
 ceph_activate_namespace() {
   kube_namespace=$1
-  CEPH_KEY=$(kubectl get secret ${PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME} \
+  secret_type=$2
-      --namespace=${PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE} \
+  secret_name=$3
-      -o json | jq -r '.data | .[]')
+  ceph_key=$4
   {
   cat <<EOF
 apiVersion: v1
 kind: Secret
 metadata:
-  name: "${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME}"
+  name: "${secret_name}"
-type: kubernetes.io/rbd
+type: "${secret_type}"
 data:
-  key: $(echo ${CEPH_KEY})
+  key: $( echo ${ceph_key} )
 EOF
   } | kubectl create --namespace ${kube_namespace} -f -
 }
 
-ceph_activate_namespace ${DEPLOYMENT_NAMESPACE}
+ceph_activate_namespace ${DEPLOYMENT_NAMESPACE} "kubernetes.io/rbd" ${PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME} "$(echo ${CEPH_RBD_KEY} | jq -r '.data | .[]')"

ceph/templates/bin/_cephfs-provisioner.sh.tpl

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+exec /usr/local/bin/cephfs-provisioner -id ${POD_NAME}

ceph/templates/configmap-bin.yaml

@@ -75,6 +75,10 @@ data:
 {{ tuple "bin/_check_zombie_mons.py.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   rbd-provisioner.sh: |
 {{ tuple "bin/_rbd-provisioner.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+  cephfs-provisioner.sh: |
+{{ tuple "bin/_cephfs-provisioner.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+  ceph-cephfs-client-key.sh: |
+{{ tuple "bin/_ceph-cephfs-client-key.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   ceph-osd-liveness-readiness.sh: |
 {{ tuple "bin/_ceph-osd-liveness-readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   ceph-mon-liveness.sh: |

ceph/templates/deployment-cephfs-provisioner.yaml

@@ -0,0 +1,175 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.deployment_cephfs_provisioner }}
+{{- $envAll := . }}
+{{- if .Values.deployment.cephfs_provisioner }}
+{{- $dependencies := .Values.dependencies.cephfs_provisioner }}
+
+{{- $serviceAccountName := "ceph-cephfs-provisioner"}}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - ''
+    resources:
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - delete
+  - apiGroups:
+      - ''
+    resources:
+      - persistentvolumeclaims
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - events
+    verbs:
+      - list
+      - watch
+      - create
+      - update
+      - patch
+  - apiGroups:
+      - ''
+    resources:
+      - services
+      - endpoints
+    verbs:
+      - get
+  - apiGroups:
+      - extensions
+    resources:
+      - podsecuritypolicies
+    resourceNames:
+      - cephfs-provisioner
+    verbs:
+      - use
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: run-cephfs-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  name: ceph-cephfs-provisioner
+spec:
+  replicas: {{ .Values.pod.replicas.cephfs_provisioner }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "cephfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      affinity:
+{{ tuple $envAll "cephfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+      initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: ceph-cephfs-provisioner
+          image: {{ .Values.images.tags.ceph_cephfs_provisioner }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.cephfs_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: PROVISIONER_NAME
+              value: {{ .Values.storageclass.rbd.provisioner }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          command:
+            - /tmp/cephfs-provisioner.sh
+          volumeMounts:
+            - name: ceph-bin
+              mountPath: /tmp/cephfs-provisioner.sh
+              subPath: cephfs-provisioner.sh
+              readOnly: true
+      volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
+{{- end }}
+{{- end }}

ceph/templates/deployment-rbd-provisioner.yaml

@@ -144,7 +144,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.rbd_provisioner | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
             - name: PROVISIONER_NAME
-              value: {{ .Values.storageclass.provisioner }}
+              value: {{ .Values.storageclass.rbd.provisioner }}
             - name: POD_NAME
               valueFrom:
                 fieldRef:

ceph/templates/job-cephfs-client-key.yaml

@@ -0,0 +1,127 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_cephfs_client_key }}
+{{- $envAll := . }}
+{{- if .Values.deployment.cephfs_provisioner }}
+{{- $dependencies := .Values.dependencies.cephfs_client_key_generator }}
+
+{{- $randStringSuffix := randAlphaNum 5 | lower }}
+
+{{- $serviceAccountName := "ceph-cephfs-client-key-generator" }}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+  namespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+  namespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-cephfs-client-key-generator
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.jobs.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name:  ceph-storage-keys-generator
+          image: {{ .Values.images.tags.ceph_config_helper }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: PVC_CEPH_CEPHFS_STORAGECLASS_USER_SECRET_NAME
+              value: {{ .Values.storageclass.cephfs.user_secret_name }}
+            - name: PVC_CEPH_CEPHFS_STORAGECLASS_ADMIN_SECRET_NAME
+              value: {{ .Values.storageclass.cephfs.admin_secret_name }}
+            - name: PVC_CEPH_CEPHFS_STORAGECLASS_DEPLOYED_NAMESPACE
+              value: {{ .Values.storageclass.cephfs.admin_secret_namespace }}
+          command:
+            - /opt/ceph/ceph-cephfs-client-key.sh
+          volumeMounts:
+            - name: ceph-bin
+              mountPath: /opt/ceph/ceph-cephfs-client-key.sh
+              subPath: ceph-cephfs-client-key.sh
+              readOnly: true
+      volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
+{{- end }}
+{{- end }}

ceph/templates/job-namespace-client-key-cleaner.yaml

@@ -83,8 +83,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-            - name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME
+            - name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
-              value: {{ .Values.storageclass.user_secret_name }}
+              value: {{ .Values.storageclass.rbd.user_secret_name }}
           command:
             - /opt/ceph/ceph-namespace-client-key-cleaner.sh
           volumeMounts:

ceph/templates/job-namespace-client-key.yaml

@@ -55,7 +55,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: Role
 metadata:
   name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
-  namespace: {{ .Values.storageclass.admin_secret_namespace }}
+  namespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
 rules:
   - apiGroups:
       - ""
@@ -69,7 +69,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: RoleBinding
 metadata:
   name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }}
-  namespace: {{ .Values.storageclass.admin_secret_namespace }}
+  namespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -105,12 +105,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-            - name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME
+            - name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME
-              value: {{ .Values.storageclass.user_secret_name }}
+              value: {{ .Values.storageclass.rbd.user_secret_name }}
-            - name: PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME
+            - name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME
-              value: {{ .Values.storageclass.admin_secret_name }}
+              value: {{ .Values.storageclass.rbd.admin_secret_name }}
-            - name: PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE
+            - name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE
-              value: {{ .Values.storageclass.admin_secret_namespace }}
+              value: {{ .Values.storageclass.rbd.admin_secret_namespace }}
           command:
             - /opt/ceph/ceph-namespace-client-key.sh
           volumeMounts:

ceph/templates/job-storage-admin-keys.yaml

@@ -85,7 +85,7 @@ spec:
             - name: CEPH_KEYRING_ADMIN_NAME
               value: {{ .Values.secrets.keyrings.admin }}
             - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME
-              value: {{ .Values.storageclass.admin_secret_name }}
+              value: {{ .Values.storageclass.rbd.admin_secret_name }}
           command:
             - /opt/ceph/ceph-storage-key.sh
           volumeMounts:

ceph/templates/storageclass-cephfs.yaml

@@ -0,0 +1,34 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.storageclass }}
+{{- $envAll := . }}
+{{- if .Values.deployment.ceph }}
+{{- if .Values.storageclass.cephfs.provision_storage_class }}
+---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: {{ .Values.storageclass.cephfs.name }}
+provisioner: {{ .Values.storageclass.cephfs.provisioner }}
+parameters:
+    monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+    adminId: {{ .Values.storageclass.cephfs.admin_id | quote }}
+    adminSecretName: {{ .Values.storageclass.cephfs.user_secret_name | quote }}
+    adminSecretNamespace: {{ .Release.Namespace | quote }}
+{{- end }}
+{{- end }}
+{{- end }}

ceph/templates/storageclass.yaml

@@ -17,23 +17,23 @@ limitations under the License.
 {{- if .Values.manifests.storageclass }}
 {{- $envAll := . }}
 {{- if .Values.deployment.ceph }}
-{{- if .Values.storageclass.provision_storage_class }}
+{{- if .Values.storageclass.rbd.provision_storage_class }}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-    name: {{ .Values.storageclass.name }}
+    name: {{ .Values.storageclass.rbd.name }}
-provisioner: {{ .Values.storageclass.provisioner }}
+provisioner: {{ .Values.storageclass.rbd.provisioner }}
 parameters:
     monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-    adminId: {{ .Values.storageclass.admin_id }}
+    adminId: {{ .Values.storageclass.rbd.admin_id }}
-    adminSecretName: {{ .Values.storageclass.admin_secret_name }}
+    adminSecretName: {{ .Values.storageclass.rbd.admin_secret_name }}
-    adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace }}
+    adminSecretNamespace: {{ .Values.storageclass.rbd.admin_secret_namespace }}
-    pool: {{ .Values.storageclass.pool }}
+    pool: {{ .Values.storageclass.rbd.pool }}
-    userId: {{ .Values.storageclass.user_id }}
+    userId: {{ .Values.storageclass.rbd.user_id }}
-    userSecretName: {{ .Values.storageclass.user_secret_name }}
+    userSecretName: {{ .Values.storageclass.rbd.user_secret_name }}
-    imageFormat: {{ .Values.storageclass.image_format | quote }}
+    imageFormat: {{ .Values.storageclass.rbd.image_format | quote }}
-    imageFeatures: {{ .Values.storageclass.image_features }}
+    imageFeatures: {{ .Values.storageclass.rbd.image_features }}
 {{- end }}
 {{- end }}
 {{- end }}

ceph/values.yaml

@@ -17,6 +17,7 @@ deployment:
   storage_secrets: true
   client_secrets: true
   rbd_provisioner: true
+  cephfs_provisioner: true
   rgw_keystone_user_and_endpoints: false
 
 images:
@@ -29,6 +30,7 @@ images:
     ceph_daemon: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04
     ceph_config_helper: docker.io/port/ceph-config-helper:v1.7.5
     ceph_rbd_provisioner: quay.io/external_storage/rbd-provisioner:v0.1.1
+    ceph_cephfs_provisioner: quay.io/external_storage/cephfs-provisioner:v0.1.1
   pull_policy: "IfNotPresent"
 
 labels:
@@ -57,13 +59,14 @@ pod:
     rgw: 1
     mon_check: 1
     rbd_provisioner: 2
+    cephfs_provisioner: 2
     mgr: 1
   affinity:
-      anti:
+    anti:
-        type:
+      type:
-          default: preferredDuringSchedulingIgnoredDuringExecution
+        default: preferredDuringSchedulingIgnoredDuringExecution
-        topologyKey:
+      topologyKey:
-          default: kubernetes.io/hostname
+        default: kubernetes.io/hostname
   resources:
     enabled: false
     osd:
@@ -108,6 +111,13 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
+    cephfs_provisioner:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
     mgr:
       requests:
         memory: "5Mi"
@@ -202,6 +212,8 @@ conf:
       mds:
 
 dependencies:
+  cephfs_client_key_generator:
+    jobs:
   job_keyring_generator:
     jobs:
   namespace_client_key_cleaner:
@@ -252,6 +264,11 @@ dependencies:
     services:
     - service: ceph_mon
       endpoint: internal
+  cephfs_provisioner:
+    jobs:
+    services:
+    - service: ceph_mon
+      endpoint: internal
   ks_user:
     services:
     - service: identity
@@ -303,18 +320,27 @@ bootstrap:
 # it is presumed you manage your own storage
 # class definition externally
 storageclass:
-  provision_storage_class: true
+  rbd:
-  provisioner: ceph.com/rbd
+    provision_storage_class: true
-  name: general
+    provisioner: ceph.com/rbd
-  monitors: null
+    name: general
-  pool: rbd
+    monitors: null
-  admin_id: admin
+    pool: rbd
-  admin_secret_name: pvc-ceph-conf-combined-storageclass
+    admin_id: admin
-  admin_secret_namespace: ceph
+    admin_secret_name: pvc-ceph-conf-combined-storageclass
-  user_id: admin
+    admin_secret_namespace: ceph
-  user_secret_name: pvc-ceph-client-key
+    user_id: admin
-  image_format: "2"
+    user_secret_name: pvc-ceph-client-key
-  image_features: layering
+    image_format: "2"
+    image_features: layering
+  cephfs:
+    provision_storage_class: true
+    provisioner: ceph.com/cephfs
+    name: cephfs
+    admin_id: admin
+    user_secret_name: pvc-ceph-cephfs-client-key
+    admin_secret_name: pvc-ceph-conf-combined-storageclass
+    admin_secret_namespace: ceph
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -388,9 +414,11 @@ manifests:
   deployment_mds: true
   deployment_moncheck: true
   deployment_rbd_provisioner: true
+  deployment_cephfs_provisioner: true
   deployment_rgw: true
   deployment_mgr: true
   job_bootstrap: true
+  job_cephfs_client_key: true
   job_keyring: true
   job_ks_endpoints: true
   job_ks_service: true

doc/source/install/developer/all-in-one.rst

@@ -13,6 +13,10 @@ to package installation.
 Requirements
 ============
 
+.. warning:: Until the Ubuntu kernel shipped with 16.04 supports CephFS
+   subvolume mounts by default the `HWE Kernel
+   <../../troubleshooting/ubuntu-hwe-kernel.rst>`__ is required to use CephFS.
+
 System Requirements
 -------------------
 

doc/source/install/multinode.rst

@@ -28,6 +28,10 @@ documentation as we explore these options further.
 The installation procedures below, will take an administrator from a new
 ``kubeadm`` installation to Openstack-Helm deployment.
 
+.. warning:: Until the Ubuntu kernel shipped with 16.04 supports CephFS
+   subvolume mounts by default the `HWE Kernel
+   <../troubleshooting/ubuntu-hwe-kernel.rst>`__ is required to use CephFS.
+
 Kubernetes Preparation
 ======================
 
@@ -581,4 +585,3 @@ The following shows a set of example hosts and the values fed into the configmap
         DEFAULT:
           vcpu_pin_set: "0-31"
           cpu_allocation_ratio: 3.0
-

doc/source/troubleshooting/index.rst

@@ -9,6 +9,7 @@ Sometimes things go wrong. These guides will help you solve many common issues w
 
    database
    persistent-storage
+   ubuntu-hwe-kernel
 
 Getting help
 ============

doc/source/troubleshooting/ubuntu-hwe-kernel.rst

@@ -0,0 +1,20 @@
+=================
+Ubuntu HWE Kernel
+=================
+
+To make use of CephFS in Ubuntu the HWE Kernel is required, until the issue
+described `here <https://github.com/kubernetes-incubator/external-storage/issues/345>`_
+is fixed.
+
+Installation
+============
+
+To deploy the HWE kernel, prior to deploying Kubernetes and OpenStack-Helm
+the following commands should be run on each node:
+
+.. code-block:: shell
+
+    #!/bin/bash
+    sudo -H apt-get update
+    sudo -H apt-get install -y linux-generic-hwe-16.04
+    sudo -H reboot now

tools/deployment/developer/04-ceph.sh

@@ -31,6 +31,7 @@ helm install --namespace=ceph ${WORK_DIR}/ceph --name=ceph \
     --set deployment.storage_secrets=true \
     --set deployment.ceph=true \
     --set deployment.rbd_provisioner=true \
+    --set deployment.cephfs_provisioner=true \
     --set deployment.client_secrets=false \
     --set deployment.rgw_keystone_user_and_endpoints=false \
     --set bootstrap.enabled=true \

tools/deployment/developer/05-ceph-ns-activate.sh

@@ -31,6 +31,7 @@ helm install --namespace=openstack ${WORK_DIR}/ceph --name=ceph-openstack-config
     --set deployment.storage_secrets=false \
     --set deployment.ceph=false \
     --set deployment.rbd_provisioner=false \
+    --set deployment.cephfs_provisioner=false \
     --set deployment.client_secrets=true \
     --set deployment.rgw_keystone_user_and_endpoints=false \
     --values=${WORK_DIR}/tools/overrides/mvp/ceph.yaml

tools/deployment/developer/10-ceph-radosgateway.sh

@@ -31,6 +31,7 @@ helm install --namespace=openstack ${WORK_DIR}/ceph --name=radosgw-openstack \
    --set deployment.storage_secrets=false \
    --set deployment.ceph=false \
    --set deployment.rbd_provisioner=false \
+   --set deployment.cephfs_provisioner=false \
    --set deployment.client_secrets=false \
    --set deployment.rgw_keystone_user_and_endpoints=true \
    --values=${WORK_DIR}/tools/overrides/mvp/ceph.yaml