Cinder: Enable backup

This PS enables the cinder backup service, for RBD, Swift and Posix based backups. Change-Id: I5b4d5a19e187784b55fa8601b244d9a45d29fb06
2017-12-27 21:55:53 -05:00 · 2017-12-27 21:55:53 -05:00 · 55f1bacf35
commit 55f1bacf35
parent 5178f3d926
17 changed files with 774 additions and 21 deletions
--- a/ceph/templates/templates/_admin.keyring.tpl
+++ b/ceph/templates/templates/_admin.keyring.tpl
@ -4,3 +4,4 @@
  caps mds = "allow"
  caps mon = "allow *"
  caps osd = "allow *"
+  caps mgr = "allow *"
--- a/cinder/templates/_funcs.tpl
+++ b/cinder/templates/_funcs.tpl
@ -24,9 +24,12 @@ true
 {{- end -}}
 {{- end -}}

-{{- define "cinder.is_ceph_backup_configured" -}}
-{{- $values := .Values.conf.cinder.DEFAULT -}}
-{{- if eq $values.backup_driver "cinder.backup.drivers.ceph" -}}
-true
+{{- define "cinder.ceph_volume_section_name" -}}
+{{- range $section, $values := .Values.conf.backends -}}
+{{- if kindIs "map" $values -}}
+{{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}}
+{{ $section }}
+{{- end -}}
+{{- end -}}
 {{- end -}}
 {{- end -}}
--- a/cinder/templates/bin/_backup-storage-init.sh.tpl
+++ b/cinder/templates/bin/_backup-storage-init.sh.tpl
@ -0,0 +1,65 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -x
+if [ "x$STORAGE_BACKEND" == "xcinder.backup.drivers.ceph" ]; then
+  SECRET=$(mktemp --suffix .yaml)
+  KEYRING=$(mktemp --suffix .keyring)
+  function cleanup {
+      rm -f ${SECRET} ${KEYRING}
+  }
+  trap cleanup EXIT
+fi
+
+set -ex
+if [ "x$STORAGE_BACKEND" == "xcinder.backup.drivers.swift" ] || \
+     [ "x$STORAGE_BACKEND" == "xcinder.backup.drivers.posix" ]; then
+  echo "INFO: no action required to use $STORAGE_BACKEND"
+elif [ "x$STORAGE_BACKEND" == "xcinder.backup.drivers.ceph" ]; then
+  ceph -s
+  function ensure_pool () {
+    ceph osd pool stats $1 || ceph osd pool create $1 $2
+    ceph osd pool application enable $1 $3
+  }
+  ensure_pool ${RBD_POOL_NAME} ${RBD_POOL_CHUNK_SIZE} "cinder-backup"
+
+  if USERINFO=$(ceph auth get client.${RBD_POOL_USER}); then
+    KEYSTR=$(echo $USERINFO | sed 's/.*\( key = .*\) caps mon.*/\1/')
+    echo $KEYSTR  > ${KEYRING}
+  else
+    #NOTE(Portdirect): Determine proper privs to assign keyring
+    ceph auth get-or-create client.${RBD_POOL_USER} \
+      mon "allow *" \
+      osd "allow *" \
+      mgr "allow *" \
+      -o ${KEYRING}
+  fi
+
+  ENCODED_KEYRING=$(sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p' ${KEYRING} | base64 -w0)
+  cat > ${SECRET} <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "${RBD_POOL_SECRET}"
+type: kubernetes.io/rbd
+data:
+  key: $( echo ${ENCODED_KEYRING} )
+EOF
+  kubectl create --namespace ${NAMESPACE} -f ${SECRET}
+
+fi
--- a/cinder/templates/bin/_ceph-keyring.sh.tpl
+++ b/cinder/templates/bin/_ceph-keyring.sh.tpl
@ -19,8 +19,8 @@ limitations under the License.
 set -ex
 export HOME=/tmp

-cat <<EOF > /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
-[client.{{ .Values.conf.backends.rbd1.rbd_user }}]
+cat <<EOF > /etc/ceph/ceph.client.${RBD_USER}.keyring
+[client.${RBD_USER}]
 {{- if .Values.conf.ceph.cinder_keyring }}
    key = {{ .Values.conf.ceph.cinder_keyring }}
 {{- else }}
--- a/cinder/templates/bin/_clean-secrets.sh.tpl
+++ b/cinder/templates/bin/_clean-secrets.sh.tpl
@ -0,0 +1,24 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+exec kubectl delete secret \
+  --namespace ${NAMESPACE} \
+  --ignore-not-found=true \
+  ${RBD_POOL_SECRET}
--- a/cinder/templates/bin/_storage-init.sh.tpl
+++ b/cinder/templates/bin/_storage-init.sh.tpl
@ -0,0 +1,62 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -x
+if [ "x$STORAGE_BACKEND" == "xcinder.volume.drivers.rbd.RBDDriver" ]; then
+  SECRET=$(mktemp --suffix .yaml)
+  KEYRING=$(mktemp --suffix .keyring)
+  function cleanup {
+      rm -f ${SECRET} ${KEYRING}
+  }
+  trap cleanup EXIT
+fi
+
+set -ex
+if [ "x$STORAGE_BACKEND" == "xcinder.volume.drivers.rbd.RBDDriver" ]; then
+  ceph -s
+  function ensure_pool () {
+    ceph osd pool stats $1 || ceph osd pool create $1 $2
+    ceph osd pool application enable $1 $3
+  }
+  ensure_pool ${RBD_POOL_NAME} ${RBD_POOL_CHUNK_SIZE} "cinder-volume"
+
+  if USERINFO=$(ceph auth get client.${RBD_POOL_USER}); then
+    KEYSTR=$(echo $USERINFO | sed 's/.*\( key = .*\) caps mon.*/\1/')
+    echo $KEYSTR  > ${KEYRING}
+  else
+    #NOTE(Portdirect): Determine proper privs to assign keyring
+    ceph auth get-or-create client.${RBD_POOL_USER} \
+      mon "allow *" \
+      osd "allow *" \
+      mgr "allow *" \
+      -o ${KEYRING}
+  fi
+
+  ENCODED_KEYRING=$(sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p' ${KEYRING} | base64 -w0)
+  cat > ${SECRET} <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "${RBD_POOL_SECRET}"
+type: kubernetes.io/rbd
+data:
+  key: $( echo ${ENCODED_KEYRING} )
+EOF
+  kubectl create --namespace ${NAMESPACE} -f ${SECRET}
+
+fi
--- a/cinder/templates/configmap-bin.yaml
+++ b/cinder/templates/configmap-bin.yaml
@ -49,6 +49,12 @@ data:
 {{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
  volume-usage-audit.sh: |+
 {{ tuple "bin/_volume-usage-audit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  backup-storage-init.sh: |+
+{{ tuple "bin/_backup-storage-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  storage-init.sh: |+
+{{ tuple "bin/_storage-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  clean-secrets.sh: |+
+{{ tuple "bin/_clean-secrets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- if .Values.bootstrap.enabled }}
  bootstrap.sh: |+
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
--- a/cinder/templates/configmap-etc.yaml
+++ b/cinder/templates/configmap-etc.yaml
@ -65,6 +65,33 @@ limitations under the License.
 {{- tuple "image" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.cinder.DEFAULT "glance_api_servers" | quote | trunc 0 -}}
 {{- end -}}

+{{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.swift" }}
+{{- if empty .Values.conf.cinder.DEFAULT.backup_swift_auth_version -}}
+{{- set .Values.conf.cinder.DEFAULT "backup_swift_auth_version" "3" | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.cinder.DEFAULT.backup_swift_auth_url -}}
+{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.cinder.DEFAULT "backup_swift_auth_url" | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.cinder.DEFAULT.backup_swift_user_domain -}}
+{{- set .Values.conf.cinder.DEFAULT "backup_swift_user_domain" .Values.endpoints.identity.auth.user.user_domain_name | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.cinder.DEFAULT.backup_swift_user -}}
+{{- set .Values.conf.cinder.DEFAULT "backup_swift_user" .Values.endpoints.identity.auth.user.username | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.cinder.DEFAULT.backup_swift_key -}}
+{{- set .Values.conf.cinder.DEFAULT "backup_swift_key" .Values.endpoints.identity.auth.user.password | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.cinder.DEFAULT.backup_swift_project_domain -}}
+{{- set .Values.conf.cinder.DEFAULT "backup_swift_project_domain" .Values.endpoints.identity.auth.user.project_domain_name | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.cinder.DEFAULT.backup_swift_project -}}
+{{- set .Values.conf.cinder.DEFAULT "backup_swift_project" .Values.endpoints.identity.auth.user.project_name | quote | trunc 0 -}}
+{{- end -}}
+{{- if empty .Values.conf.cinder.DEFAULT.swift_catalog_info -}}
+{{- set .Values.conf.cinder.DEFAULT "swift_catalog_info" "object-store:swift:internalURL" | quote | trunc 0 -}}
+{{- end -}}
+{{- end -}}
+
 ---
 apiVersion: v1
 kind: ConfigMap
--- a/cinder/templates/deployment-backup.yaml
+++ b/cinder/templates/deployment-backup.yaml
@ -46,7 +46,30 @@ spec:
        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
      initContainers:
 {{ tuple $envAll $dependencies $mounts_cinder_backup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
-        {{- if include "cinder.is_ceph_backup_configured" . }}
+        {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph" }}
+        - name: ceph-backup-keyring-placement
+          image: {{ .Values.images.tags.cinder_backup }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - /tmp/ceph-keyring.sh
+          env:
+            - name: RBD_USER
+              value: {{ .Values.conf.cinder.DEFAULT.backup_ceph_user | quote }}
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: cinder-bin
+              mountPath: /tmp/ceph-keyring.sh
+              subPath: ceph-keyring.sh
+              readOnly: true
+            - name: ceph-backup-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+        {{ end }}
+        {{- if include "cinder.is_ceph_volume_configured" $envAll }}
        - name: ceph-keyring-placement
          image: {{ .Values.images.tags.cinder_backup }}
          imagePullPolicy: {{ .Values.images.pull_policy }}
@ -54,6 +77,9 @@ spec:
            runAsUser: 0
          command:
            - /tmp/ceph-keyring.sh
+          env:
+            - name: RBD_USER
+              value: {{ index (index .Values.conf.backends (include "cinder.ceph_volume_section_name" $envAll)) "rbd_user" | quote }}
          volumeMounts:
            - name: etcceph
              mountPath: /etc/ceph
@ -66,6 +92,21 @@ spec:
              subPath: key
              readOnly: true
        {{ end }}
+        {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.posix" }}
+        - name: ceph-backup-volume-perms
+          image: {{ .Values.images.tags.cinder_backup }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - chown
+            - -R
+            - "cinder:"
+            - {{ .Values.conf.cinder.DEFAULT.backup_posix_path }}
+          volumeMounts:
+            - name: cinder-backup
+              mountPath: {{ .Values.conf.cinder.DEFAULT.backup_posix_path }}
+        {{ end }}
      containers:
        - name: cinder-backup
          image: {{ .Values.images.tags.cinder_backup }}
@ -74,7 +115,8 @@ spec:
          securityContext:
            runAsUser: {{ .Values.pod.user.cinder.uid }}
            capabilities:
-              add: ["SYS_ADMIN"]
+              add:
+                - SYS_ADMIN
          command:
            - /tmp/cinder-backup.sh
          volumeMounts:
@ -86,7 +128,7 @@ spec:
              mountPath: /etc/cinder/cinder.conf
              subPath: cinder.conf
              readOnly: true
-            {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph" }}
+            {{ if or (eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph") (include "cinder.is_ceph_volume_configured" $envAll) }}
            - name: etcceph
              mountPath: /etc/ceph
            - name: ceph-etc
@ -98,6 +140,10 @@ spec:
              subPath: key
              readOnly: true
            {{- end }}
+            {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.posix" }}
+            - name: cinder-backup
+              mountPath: {{ .Values.conf.cinder.DEFAULT.backup_posix_path }}
+            {{- end }}
            - name: cinder-etc
              # NOTE (Portdirect): We mount here to override Kollas
              # custom sudoers file when using Kolla images, this
@ -127,16 +173,28 @@ spec:
          configMap:
            name: cinder-bin
            defaultMode: 0555
-        {{- if include "cinder.is_ceph_backup_configured" . }}
+        {{ if or (eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph") (include "cinder.is_ceph_volume_configured" $envAll) }}
        - name: etcceph
          emptyDir: {}
        - name: ceph-etc
          configMap:
            name: ceph-etc
            defaultMode: 0444
+        {{ end }}
+        {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph" }}
+        - name: ceph-backup-keyring
+          secret:
+            secretName: {{ .Values.secrets.rbd.backup | quote }}
+        {{ end }}
+        {{- if include "cinder.is_ceph_volume_configured" $envAll }}
        - name: ceph-keyring
          secret:
-            secretName: pvc-ceph-client-key
+            secretName: {{ .Values.secrets.rbd.volume | quote }}
        {{ end }}
+        {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.posix" }}
+        - name: cinder-backup
+          persistentVolumeClaim:
+            claimName: cinder-backup
+        {{- end }}
 {{ if $mounts_cinder_backup.volumes }}{{ toYaml $mounts_cinder_backup.volumes | indent 8 }}{{ end }}
 {{- end }}
--- a/cinder/templates/deployment-volume.yaml
+++ b/cinder/templates/deployment-volume.yaml
@ -46,7 +46,7 @@ spec:
        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
      initContainers:
 {{ tuple $envAll $dependencies $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
-        {{- if include "cinder.is_ceph_volume_configured" . }}
+        {{- if include "cinder.is_ceph_volume_configured" $envAll }}
        - name: ceph-keyring-placement
          image: {{ .Values.images.tags.cinder_volume }}
          imagePullPolicy: {{ .Values.images.pull_policy }}
@ -54,6 +54,9 @@ spec:
            runAsUser: 0
          command:
            - /tmp/ceph-keyring.sh
+          env:
+            - name: RBD_USER
+              value: {{ index (index .Values.conf.backends (include "cinder.ceph_volume_section_name" $envAll)) "rbd_user" | quote }}
          volumeMounts:
            - name: etcceph
              mountPath: /etc/ceph
@ -88,7 +91,7 @@ spec:
              mountPath: /etc/cinder/conf/backends.conf
              subPath: backends.conf
              readOnly: true
-            {{- if include "cinder.is_ceph_volume_configured" . }}
+            {{- if include "cinder.is_ceph_volume_configured" $envAll }}
            - name: etcceph
              mountPath: /etc/ceph
            - name: ceph-etc
@ -128,7 +131,7 @@ spec:
        - name: cinder-etc
          configMap:
            name: cinder-etc
-        {{- if include "cinder.is_ceph_volume_configured" . }}
+        {{- if include "cinder.is_ceph_volume_configured" $envAll }}
        - name: etcceph
          emptyDir: {}
        - name: ceph-etc
@ -137,7 +140,7 @@ spec:
            defaultMode: 0444
        - name: ceph-keyring
          secret:
-            secretName: pvc-ceph-client-key
+            secretName: {{ .Values.secrets.rbd.volume | quote }}
        {{ end }}
 {{ if $mounts_cinder_volume.volumes }}{{ toYaml $mounts_cinder_volume.volumes | indent 8 }}{{ end }}
 {{- end }}
--- a/cinder/templates/job-backup-storage-init.yaml
+++ b/cinder/templates/job-backup-storage-init.yaml
@ -0,0 +1,147 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_backup_storage_init }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.backup_storage_init }}
+
+{{- $serviceAccountName := "cinder-backup-storage-init" }}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cinder-backup-storage-init
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "cinder" "storage-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph" }}
+        - name: ceph-keyring-placement
+          image: {{ .Values.images.tags.cinder_backup_storage_init }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - /tmp/ceph-keyring.sh
+          env:
+            - name: RBD_USER
+              value: "admin"
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: cinder-bin
+              mountPath: /tmp/ceph-keyring.sh
+              subPath: ceph-keyring.sh
+              readOnly: true
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+        {{ end }}
+      containers:
+        - name: cinder-backup-storage-init
+          image: {{ .Values.images.tags.cinder_backup_storage_init }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.backup_storage_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: STORAGE_BACKEND
+              value: {{ .Values.conf.cinder.DEFAULT.backup_driver | quote }}
+            {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph" }}
+            - name: RBD_POOL_NAME
+              value: {{ .Values.conf.cinder.DEFAULT.backup_ceph_pool | quote }}
+            - name: RBD_POOL_USER
+              value: {{ .Values.conf.cinder.DEFAULT.backup_ceph_user | quote }}
+            - name: RBD_POOL_CHUNK_SIZE
+              value: "8"
+            - name: RBD_POOL_SECRET
+              value: {{ .Values.secrets.rbd.backup | quote }}
+            {{ end }}
+          command:
+            - /tmp/backup-storage-init.sh
+          volumeMounts:
+            - name: cinder-bin
+              mountPath: /tmp/backup-storage-init.sh
+              subPath: backup-storage-init.sh
+              readOnly: true
+            {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph" }}
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-etc
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+            {{- end }}
+      volumes:
+        - name: cinder-bin
+          configMap:
+            name: cinder-bin
+            defaultMode: 0555
+        {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph" }}
+        - name: etcceph
+          emptyDir: {}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+            defaultMode: 0444
+        - name: ceph-keyring
+          secret:
+            secretName: pvc-ceph-client-key
+        {{ end }}
+{{- end }}
--- a/cinder/templates/job-clean.yaml
+++ b/cinder/templates/job-clean.yaml
@ -0,0 +1,123 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_clean }}
+{{- $envAll := . }}
+{{ if or (eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph") (include "cinder.is_ceph_volume_configured" $envAll) }}
+{{- $dependencies := .Values.dependencies.clean }}
+
+{{- $randStringSuffix := randAlphaNum 5 | lower }}
+
+{{- $serviceAccountName := print "cinder-clean-" $randStringSuffix }}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ print "cinder-clean-" $randStringSuffix }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "cinder" "clean" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        {{- if include "cinder.is_ceph_volume_configured" $envAll }}
+        - name: cinder-volume-rbd-secret-clean
+          image: {{ .Values.images.tags.cinder_storage_init }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.clean | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: RBD_POOL_SECRET
+              value: {{ .Values.secrets.rbd.volume | quote }}
+          command:
+            - /tmp/clean-secrets.sh
+          volumeMounts:
+            - name: cinder-bin
+              mountPath: /tmp/clean-secrets.sh
+              subPath: clean-secrets.sh
+              readOnly: true
+        {{ end }}
+        {{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.ceph" }}
+        - name: cinder-volume-backup-secret-clean
+          image: {{ .Values.images.tags.cinder_backup_storage_init }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.clean | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: RBD_POOL_SECRET
+              value: {{ .Values.secrets.rbd.backup | quote }}
+          command:
+            - /tmp/clean-secrets.sh
+          volumeMounts:
+            - name: cinder-bin
+              mountPath: /tmp/clean-secrets.sh
+              subPath: clean-secrets.sh
+              readOnly: true
+        {{ end }}
+      volumes:
+        - name: cinder-bin
+          configMap:
+            name: cinder-bin
+            defaultMode: 0555
+{{- end }}
+{{- end }}
--- a/cinder/templates/job-storage-init.yaml
+++ b/cinder/templates/job-storage-init.yaml
@ -0,0 +1,147 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_storage_init }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.storage_init }}
+
+{{- $serviceAccountName := "cinder-storage-init" }}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cinder-storage-init
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "cinder" "storage-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        {{- if include "cinder.is_ceph_volume_configured" $envAll }}
+        - name: ceph-keyring-placement
+          image: {{ .Values.images.tags.cinder_storage_init }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - /tmp/ceph-keyring.sh
+          env:
+            - name: RBD_USER
+              value: "admin"
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: cinder-bin
+              mountPath: /tmp/ceph-keyring.sh
+              subPath: ceph-keyring.sh
+              readOnly: true
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+        {{ end }}
+      containers:
+        - name: cinder-storage-init
+          image: {{ .Values.images.tags.cinder_storage_init }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.storage_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            {{- if include "cinder.is_ceph_volume_configured" $envAll }}
+            - name: STORAGE_BACKEND
+              value: {{ index (index .Values.conf.backends (include "cinder.ceph_volume_section_name" $envAll)) "volume_driver" | quote }}
+            - name: RBD_POOL_NAME
+              value: {{ index (index .Values.conf.backends (include "cinder.ceph_volume_section_name" $envAll)) "rbd_pool" | quote }}
+            - name: RBD_POOL_USER
+              value: {{ index (index .Values.conf.backends (include "cinder.ceph_volume_section_name" $envAll)) "rbd_user" | quote }}
+            - name: RBD_POOL_CHUNK_SIZE
+              value: "8"
+            - name: RBD_POOL_SECRET
+              value: {{ .Values.secrets.rbd.volume | quote }}
+            {{- end }}
+          command:
+            - /tmp/storage-init.sh
+          volumeMounts:
+            - name: cinder-bin
+              mountPath: /tmp/storage-init.sh
+              subPath: storage-init.sh
+              readOnly: true
+            {{- if include "cinder.is_ceph_volume_configured" $envAll }}
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-etc
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+            {{- end }}
+      volumes:
+        - name: cinder-bin
+          configMap:
+            name: cinder-bin
+            defaultMode: 0555
+        {{- if include "cinder.is_ceph_volume_configured" $envAll }}
+        - name: etcceph
+          emptyDir: {}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+            defaultMode: 0444
+        - name: ceph-keyring
+          secret:
+            secretName: pvc-ceph-client-key
+        {{- end }}
+{{- end }}
--- a/cinder/templates/pvc-backup.yaml
+++ b/cinder/templates/pvc-backup.yaml
@ -0,0 +1,32 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.pvc_backup }}
+{{- $envAll := . }}
+{{- if eq .Values.conf.cinder.DEFAULT.backup_driver "cinder.backup.drivers.posix" }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: cinder-backup
+spec:
+  accessModes: [ "ReadWriteOnce" ]
+  resources:
+    requests:
+      storage: {{ .Values.backup.posix.volume.size }}
+  storageClassName: {{ .Values.backup.posix.volume.class_name }}
+{{- end }}
+{{- end }}
--- a/cinder/values.yaml
+++ b/cinder/values.yaml
@ -39,7 +39,9 @@ images:
    cinder_scheduler: docker.io/kolla/ubuntu-source-cinder-scheduler:3.0.3
    cinder_volume: docker.io/kolla/ubuntu-source-cinder-volume:3.0.3
    cinder_volume_usage_audit: docker.io/kolla/ubuntu-source-cinder-volume:3.0.3
+    cinder_storage_init: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04
    cinder_backup: docker.io/kolla/ubuntu-source-cinder-backup:3.0.3
+    cinder_backup_storage_init: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04
    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
  pull_policy: "IfNotPresent"

@ -154,6 +156,27 @@ pod:
        limits:
          memory: "1024Mi"
          cpu: "2000m"
+      clean:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      backup_storage_init:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      storage_init:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
      ks_endpoints:
        requests:
          memory: "128Mi"
@ -395,10 +418,15 @@ conf:
      host: cinder-volume-worker
      osapi_volume_listen_port: 8776
      enabled_backends: "rbd1"
+      # NOTE(portdirect): "cinder.backup.drivers.swift"  and
+      # "cinder.backup.drivers.posix" also supported
      backup_driver: "cinder.backup.drivers.ceph"
+      # Backup: Ceph RBD options
      backup_ceph_conf: "/etc/ceph/ceph.conf"
-      backup_ceph_user: admin
-      backup_ceph_pool: backups
+      backup_ceph_user: cinderbackup
+      backup_ceph_pool: cinder.backups
+      # Backup: Posix options
+      backup_posix_path: /var/lib/cinder/backup
    database:
      max_retries: -1
    keystone_authtoken:
@ -414,13 +442,13 @@ conf:
    rbd1:
      volume_driver: cinder.volume.drivers.rbd.RBDDriver
      volume_backend_name: rbd1
-      rbd_pool: volumes
+      rbd_pool: cinder.volumes
      rbd_ceph_conf: "/etc/ceph/ceph.conf"
      rbd_flatten_volume_from_snapshot: false
      rbd_max_clone_depth: 5
      rbd_store_chunk_size: 4
      rados_connect_timeout: -1
-      rbd_user: "admin"
+      rbd_user: cinder
  rally_tests:
    run_tempest: false
    tests:
@ -454,6 +482,12 @@ conf:
            failure_rate:
              max: 0

+backup:
+  posix:
+    volume:
+      class_name: general
+      size: 10Gi
+
 dependencies:
  db_init:
    services:
@ -483,11 +517,18 @@ dependencies:
    services:
    - service: identity
      endpoint: internal
+  backup_storage_init:
+    jobs:
+  storage_init:
+    jobs:
+  clean:
+    jobs:
  api:
    jobs:
    - cinder-db-sync
    - cinder-ks-user
    - cinder-ks-endpoints
+    - cinder-storage-init
    services:
    - service: oslo_db
      endpoint: internal
@ -504,6 +545,7 @@ dependencies:
    - cinder-db-sync
    - cinder-ks-user
    - cinder-ks-endpoints
+    - cinder-storage-init
    services:
    - service: identity
      endpoint: internal
@ -514,6 +556,7 @@ dependencies:
    - cinder-db-sync
    - cinder-ks-user
    - cinder-ks-endpoints
+    - cinder-storage-init
    services:
    - service: identity
      endpoint: internal
@ -524,6 +567,7 @@ dependencies:
    - cinder-db-sync
    - cinder-ks-user
    - cinder-ks-endpoints
+    - cinder-storage-init
    services:
    - service: identity
      endpoint: internal
@ -534,6 +578,8 @@ dependencies:
    - cinder-db-sync
    - cinder-ks-user
    - cinder-ks-endpoints
+    - cinder-storage-init
+    - cinder-backup-storage-init
    services:
    - service: identity
      endpoint: internal
@ -554,6 +600,9 @@ secrets:
  oslo_db:
    admin: cinder-db-admin
    user: cinder-db-user
+  rbd:
+    backup: cinder-backup-rbd-keyring
+    volume: cinder-volume-rbd-keyring

 # We use a different layout of the endpoints here to account for versioning
 # this swaps the service name and type, and should be rolled out to other
@ -719,15 +768,19 @@ manifests:
  deployment_scheduler: true
  deployment_volume: true
  ingress_api: true
+  job_backup_storage_init: true
  job_bootstrap: true
+  job_clean: true
  job_db_init: true
  job_db_sync: true
  job_db_drop: false
  job_ks_endpoints: true
  job_ks_service: true
  job_ks_user: true
+  job_storage_init: true
  pdb_api: true
  pod_rally_test: true
+  pvc_backup: true
  secret_db: true
  secret_keystone: true
  service_api: true
--- a/horizon/templates/etc/_local_settings.tpl
+++ b/horizon/templates/etc/_local_settings.tpl
@ -244,7 +244,7 @@ OPENSTACK_HYPERVISOR_FEATURES = {
 # The OPENSTACK_CINDER_FEATURES settings can be used to enable optional
 # services provided by cinder that is not exposed by its extension API.
 OPENSTACK_CINDER_FEATURES = {
-    'enable_backup': False,
+    'enable_backup': {{ .Values.local_settings.openstack_cinder_features.enable_backup }},
 }

 # The OPENSTACK_NEUTRON_NETWORK settings can be used to enable optional
--- a/horizon/values.yaml
+++ b/horizon/values.yaml
@ -46,6 +46,8 @@ network:
 local_settings:
  horizon_secret_key: 9aee62c0-5253-4a86-b189-e0fb71fa503c
  debug: "True"
+  openstack_cinder_features:
+    enable_backup: "True"
  openstack_neutron_network:
    enable_router: "True"
    enable_quotas: "True"