Add cinder_csi_enabled label

Add support for out of tree Cinder CSI. This is installed when the cinder_csi_enabled=true label is added. This will allow us to eventually deprecate in-tree Cinder. story: 2007048 task: 37868 Change-Id: I8305b9f8c9c37518ec39198693adb6f18542bf2e Signed-off-by: Bharat Kunwar <brtknr@bath.edu>
2019-12-22 21:42:48 +00:00 · 2019-12-22 21:42:48 +00:00 · 9565984fd9
commit 9565984fd9
parent fb283f595a
11 changed files with 794 additions and 1 deletions
--- a/doc/source/user/index.rst
+++ b/doc/source/user/index.rst
@ -1199,6 +1199,14 @@ _`container_infra_prefix`
  * docker.io/planetlabs/draino:abf028a
  * docker.io/openstackmagnum/cluster-autoscaler:v1.0

+  Images that might be needed if 'cinder_csi_enabled' is 'true':
+  * docker.io/k8scloudprovider/cinder-csi-plugin:v1.16.0
+  * quay.io/k8scsi/csi-attacher:v2.0.0
+  * quay.io/k8scsi/csi-provisioner:v1.4.0
+  * quay.io/k8scsi/csi-snapshotter:v1.2.2
+  * quay.io/k8scsi/csi-resizer:v0.3.0
+  * quay.io/k8scsi/csi-node-driver-registrar:v1.1.0
+
 _`kube_tag`
  This label allows users to select `a specific Kubernetes release,
  based on its container tag
@ -1328,6 +1336,49 @@ _`cloud_provider_enabled`
  if 'cinder' is selected as a 'volume_driver', it is implied that the cloud
  provider will be enabled since they are combined.

+_`cinder_csi_enabled`
+  When 'true', out-of-tree Cinder CSI driver will be enabled. Requires 'cinder'
+  to be selected as a 'volume_driver' and consequently also requires label
+  'cloud_provider_enabled' to be 'true' (see 'cloud_provider_enabled' section).
+  Default: false
+
+_`cinder_csi_plugin_tag`
+  This label allows users to override the default container tag for Cinder CSI plugin.
+  For additional tags, `refer to Cinder CSI plugin page
+  <https://hub.docker.com/r/k8scloudprovider/cinder-csi-plugin/tags>`_.
+  Ussuri-default: v1.16.0
+
+_`csi_attacher_tag`
+  This label allows users to override the default container tag for CSI attacher.
+  For additional tags, `refer to CSI attacher page
+  <https://quay.io/repository/k8scsi/csi-attacher?tab=tags>`_.
+  Ussuri-default: v2.0.0
+
+_`csi_provisioner_tag`
+  This label allows users to override the default container tag for CSI provisioner.
+  For additional tags, `refer to CSI provisioner page
+  <https://quay.io/repository/k8scsi/csi-provisioner?tab=tags>`_.
+  Ussuri-default: v1.4.0
+
+_`csi_snapshotter_tag`
+  This label allows users to override the default container tag for CSI snapshotter.
+  For additional tags, `refer to CSI snapshotter page
+  <https://quay.io/repository/k8scsi/csi-snapshotter?tab=tags>`_.
+  Ussuri-default: v1.2.2
+
+_`csi_resizer_tag`
+  This label allows users to override the default container tag for CSI resizer.
+  For additional tags, `refer to CSI resizer page
+  <https://quay.io/repository/k8scsi/csi-resizer?tab=tags>`_.
+  Ussuri-default: v0.3.0
+
+_`csi_node_driver_registrar_tag`
+  This label allows users to override the default container tag for CSI node
+  driver registrar. For additional tags, `refer to CSI node driver registrar
+  page
+  <https://quay.io/repository/k8scsi/csi-node-driver-registrar?tab=tags>`_.
+  Ussuri-default: v1.1.0
+
 _`keystone_auth_enabled`
  If this label is set to True, Kubernetes will support use Keystone for
  authorization and authentication.
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
@ -386,7 +386,9 @@ fi

 if [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
    KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-provider=external"
-    KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --external-cloud-volume-plugin=openstack --cloud-config=/etc/kubernetes/cloud-config"
+    if [ "$(echo "${VOLUME_DRIVER}" | tr '[:upper:]' '[:lower:]')" = "cinder" ] && [ "$(echo "${CINDER_CSI_ENABLED}" | tr '[:upper:]' '[:lower:]')" != "true" ]; then
+        KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --external-cloud-volume-plugin=openstack --cloud-config=/etc/kubernetes/cloud-config"
+    fi
 fi


--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-cinder-csi.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-cinder-csi.sh
@ -0,0 +1,521 @@
+#!/bin/sh
+
+step="enable-cinder-csi"
+printf "Starting to run ${step}\n"
+
+. /etc/sysconfig/heat-params
+
+volume_driver=$(echo "${VOLUME_DRIVER}" | tr '[:upper:]' '[:lower:]')
+cinder_csi_enabled=$(echo $CINDER_CSI_ENABLED | tr '[:upper:]' '[:lower:]')
+
+if [ "${volume_driver}" = "cinder" ] && [ "${cinder_csi_enabled}" = "true" ]; then
+    # Generate Cinder CSI manifest file
+    CINDER_CSI_DEPLOY=/srv/magnum/kubernetes/manifests/cinder-csi.yaml
+    echo "Writing File: $CINDER_CSI_DEPLOY"
+    mkdir -p $(dirname ${CINDER_CSI_DEPLOY})
+    cat << EOF > ${CINDER_CSI_DEPLOY}
+---
+# This YAML file contains RBAC API objects,
+# which are necessary to run csi controller plugin
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-cinder-controller-sa
+  namespace: kube-system
+---
+# external attacher
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+---
+# external Provisioner
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-provisioner-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+---
+# external snapshotter
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "list", "watch", "delete"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-snapshotter-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+---
+# External Resizer
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-role
+rules:
+  # The following rule should be uncommented for plugins that require secrets
+  # for provisioning.
+  # - apiGroups: [""]
+  #   resources: ["secrets"]
+  #   verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: kube-system
+  name: external-resizer-cfg
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-role-cfg
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: external-resizer-cfg
+  apiGroup: rbac.authorization.k8s.io
+---
+# This YAML file contains CSI Controller Plugin Sidecars
+# external-attacher, external-provisioner, external-snapshotter
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-cinder-controller-service
+  namespace: kube-system
+  labels:
+    app: csi-cinder-controllerplugin
+spec:
+  selector:
+    app: csi-cinder-controllerplugin
+  ports:
+    - name: dummy
+      port: 12345
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-cinder-controllerplugin
+  namespace: kube-system
+spec:
+  serviceName: "csi-cinder-controller-service"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-cinder-controllerplugin
+  template:
+    metadata:
+      labels:
+        app: csi-cinder-controllerplugin
+    spec:
+      serviceAccount: csi-cinder-controller-sa
+      containers:
+        - name: csi-attacher
+          image: ${CONTAINER_INFRA_PREFIX:-quay.io/k8scsi/}csi-attacher:${CSI_ATTACHER_TAG}
+          args:
+            - "--v=5"
+            - "--csi-address=\$(ADDRESS)"
+            - "--timeout=3m"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-provisioner
+          image: ${CONTAINER_INFRA_PREFIX:-quay.io/k8scsi/}csi-provisioner:${CSI_PROVISIONER_TAG}
+          args:
+            - "--csi-address=\$(ADDRESS)"
+            - "--timeout=3m"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-snapshotter
+          image: ${CONTAINER_INFRA_PREFIX:-quay.io/k8scsi/}csi-snapshotter:${CSI_SNAPSHOTTER_TAG}
+          args:
+            - "--csi-address=\$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - mountPath: /var/lib/csi/sockets/pluginproxy/
+              name: socket-dir
+        - name: csi-resizer
+          image: ${CONTAINER_INFRA_PREFIX:-quay.io/k8scsi/}csi-resizer:${CSI_RESIZER_TAG}
+          args:
+            - "--v=5"
+            - "--csi-address=\$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: cinder-csi-plugin
+          image: ${CONTAINER_INFRA_PREFIX:-docker.io/k8scloudprovider/}cinder-csi-plugin:${CINDER_CSI_PLUGIN_TAG}
+          args :
+            - /bin/cinder-csi-plugin
+            - "--nodeid=\$(NODE_ID)"
+            - "--endpoint=\$(CSI_ENDPOINT)"
+            - "--cloud-config=\$(CLOUD_CONFIG)"
+            - "--cluster=\$(CLUSTER_NAME)"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix://csi/csi.sock
+            - name: CLOUD_CONFIG
+              value: /etc/config/cloud-config
+            - name: CLUSTER_NAME
+              value: kubernetes
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: secret-cinderplugin
+              mountPath: /etc/config
+              readOnly: true
+            - name: cacert
+              mountPath: /etc/kubernetes/ca-bundle.crt
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          emptyDir:
+        - name: secret-cinderplugin
+          secret:
+            secretName: cinder-csi-cloud-config
+        - name: cacert
+          hostPath:
+            path: /etc/kubernetes/ca-bundle.crt
+            type: File
+---
+# This YAML defines all API objects to create RBAC roles for csi node plugin.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-cinder-node-sa
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nodeplugin-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nodeplugin-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-cinder-node-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-nodeplugin-role
+  apiGroup: rbac.authorization.k8s.io
+---
+# This YAML file contains driver-registrar & csi driver nodeplugin API objects,
+# which are necessary to run csi nodeplugin for cinder.
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-cinder-nodeplugin
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: csi-cinder-nodeplugin
+  template:
+    metadata:
+      labels:
+        app: csi-cinder-nodeplugin
+    spec:
+      serviceAccount: csi-cinder-node-sa
+      hostNetwork: true
+      containers:
+        - name: node-driver-registrar
+          image: ${CONTAINER_INFRA_PREFIX:-quay.io/k8scsi/}csi-node-driver-registrar:${CSI_NODE_DRIVER_REGISTRAR_TAG}
+          args:
+            - "--csi-address=\$(ADDRESS)"
+            - "--kubelet-registration-path=\$(DRIVER_REG_SOCK_PATH)"
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/cinder.csi.openstack.org /registration/cinder.csi.openstack.org-reg.sock"]
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: cinder-csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: ${CONTAINER_INFRA_PREFIX:-docker.io/k8scloudprovider/}cinder-csi-plugin:${CINDER_CSI_PLUGIN_TAG}
+          args :
+            - /bin/cinder-csi-plugin
+            - "--nodeid=\$(NODE_ID)"
+            - "--endpoint=\$(CSI_ENDPOINT)"
+            - "--cloud-config=\$(CLOUD_CONFIG)"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix://csi/csi.sock
+            - name: CLOUD_CONFIG
+              value: /etc/config/cloud-config
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: pods-cloud-data
+              mountPath: /var/lib/cloud/data
+              readOnly: true
+            - name: pods-probe-dir
+              mountPath: /dev
+              mountPropagation: "HostToContainer"
+            - name: secret-cinderplugin
+              mountPath: /etc/config
+              readOnly: true
+            - name: cacert
+              mountPath: /etc/kubernetes/ca-bundle.crt
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/cinder.csi.openstack.org
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: pods-cloud-data
+          hostPath:
+            path: /var/lib/cloud/data
+        - name: pods-probe-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: secret-cinderplugin
+          secret:
+            secretName: cinder-csi-cloud-config
+        - name: cacert
+          hostPath:
+            path: /etc/kubernetes/ca-bundle.crt
+            type: File
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: cinder.csi.openstack.org
+spec:
+  attachRequired: true
+  podInfoOnMount: true
+  volumeLifecycleModes:
+  - Persistent
+  - Ephemeral
+EOF
+
+    echo "Waiting for Kubernetes API..."
+    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    do
+        sleep 5
+    done
+
+    cat <<EOF | kubectl apply -f -
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cinder-csi-cloud-config
+  namespace: kube-system
+type: Opaque
+stringData:
+  cloud-config: |-
+    [Global]
+    auth-url=$AUTH_URL
+    user-id=$TRUSTEE_USER_ID
+    password=$TRUSTEE_PASSWORD
+    trust-id=$TRUST_ID
+    region=$REGION_NAME
+    ca-file=/etc/kubernetes/ca-bundle.crt
+EOF
+
+    kubectl apply -f ${CINDER_CSI_DEPLOY}
+fi
+printf "Finished running ${step}\n"
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh
@ -108,6 +108,13 @@ NGINX_INGRESS_CONTROLLER_CHART_TAG="$NGINX_INGRESS_CONTROLLER_CHART_TAG"
 AUTO_HEALING_ENABLED="$AUTO_HEALING_ENABLED"
 AUTO_HEALING_CONTROLLER="$AUTO_HEALING_CONTROLLER"
 AUTO_SCALING_ENABLED="$AUTO_SCALING_ENABLED"
+CINDER_CSI_ENABLED="$CINDER_CSI_ENABLED"
+CINDER_CSI_PLUGIN_TAG="$CINDER_CSI_PLUGIN_TAG"
+CSI_ATTACHER_TAG="$CSI_ATTACHER_TAG"
+CSI_PROVISIONER_TAG="$CSI_PROVISIONER_TAG"
+CSI_SNAPSHOTTER_TAG="$CSI_SNAPSHOTTER_TAG"
+CSI_RESIZER_TAG="$CSI_RESIZER_TAG"
+CSI_NODE_DRIVER_REGISTRAR_TAG="$CSI_NODE_DRIVER_REGISTRAR_TAG"
 DRAINO_TAG="$DRAINO_TAG"
 MAGNUM_AUTO_HEALER_TAG="$MAGNUM_AUTO_HEALER_TAG"
 AUTOSCALER_TAG="$AUTOSCALER_TAG"
--- a/magnum/drivers/heat/k8s_fedora_template_def.py
+++ b/magnum/drivers/heat/k8s_fedora_template_def.py
@ -81,6 +81,10 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
                      'calico_tag',
                      'calico_kube_controllers_tag', 'calico_ipv4pool',
                      'calico_ipv4pool_ipip',
+                      'cinder_csi_enabled', 'cinder_csi_plugin_tag',
+                      'csi_attacher_tag', 'csi_provisioner_tag',
+                      'csi_snapshotter_tag', 'csi_resizer_tag',
+                      'csi_node_driver_registrar_tag',
                      'etcd_tag', 'flannel_tag', 'flannel_cni_tag',
                      'cloud_provider_tag',
                      'prometheus_tag', 'grafana_tag',
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
@ -748,6 +748,43 @@ parameters:
    default:
      false

+  cinder_csi_enabled:
+    type: boolean
+    description: >
+      true if the cinder csi feature should be enabled
+    default:
+      false
+
+  cinder_csi_plugin_tag:
+    type: string
+    description: tag of cinder csi plugin
+    default: v1.16.0
+
+  csi_attacher_tag:
+    type: string
+    description: tag of csi attacher
+    default: v2.0.0
+
+  csi_provisioner_tag:
+    type: string
+    description: tag of csi provisioner
+    default: v1.4.0
+
+  csi_snapshotter_tag:
+    type: string
+    description: tag of csi snapshotter
+    default: v1.2.2
+
+  csi_resizer_tag:
+    type: string
+    description: tag of csi resizer
+    default: v0.3.0
+
+  csi_node_driver_registrar_tag:
+    type: string
+    description: tag of csi node driver registrar
+    default: v1.1.0
+
  node_problem_detector_tag:
    type: string
    description: tag of the node problem detector container
@ -1137,6 +1174,13 @@ resources:
          auto_healing_controller: {get_param: auto_healing_controller}
          magnum_auto_healer_tag: {get_param: magnum_auto_healer_tag}
          auto_scaling_enabled: {get_param: auto_scaling_enabled}
+          cinder_csi_enabled: {get_param: cinder_csi_enabled}
+          cinder_csi_plugin_tag: {get_param: cinder_csi_plugin_tag}
+          csi_attacher_tag: {get_param: csi_attacher_tag}
+          csi_provisioner_tag: {get_param: csi_provisioner_tag}
+          csi_snapshotter_tag: {get_param: csi_snapshotter_tag}
+          csi_resizer_tag: {get_param: csi_resizer_tag}
+          csi_node_driver_registrar_tag: {get_param: csi_node_driver_registrar_tag}
          draino_tag: {get_param: draino_tag}
          autoscaler_tag: {get_param: autoscaler_tag}
          min_node_count: {get_param: min_node_count}
@ -1176,6 +1220,7 @@ resources:
                  "$KEYSTONE_AUTH_DEFAULT_POLICY": {get_param: keystone_auth_default_policy}
            - get_file: ../../common/templates/kubernetes/fragments/enable-auto-healing.sh
            - get_file: ../../common/templates/kubernetes/fragments/enable-auto-scaling.sh
+            - get_file: ../../common/templates/kubernetes/fragments/enable-cinder-csi.sh
            # Helm Based Installation Configuration Scripts
            - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
            - str_replace:
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
@ -525,6 +525,35 @@ parameters:
    description: >
      true if the auto scaling feature should be enabled

+  cinder_csi_enabled:
+    type: boolean
+    description: >
+      true if the cinder csi feature should be enabled
+
+  cinder_csi_plugin_tag:
+    type: string
+    description: tag of cinder csi plugin
+
+  csi_attacher_tag:
+    type: string
+    description: tag of csi attacher
+
+  csi_provisioner_tag:
+    type: string
+    description: tag of csi provisioner
+
+  csi_snapshotter_tag:
+    type: string
+    description: tag of csi snapshotter
+
+  csi_resizer_tag:
+    type: string
+    description: tag of csi resizer
+
+  csi_node_driver_registrar_tag:
+    type: string
+    description: tag of csi node driver registrar
+
  node_problem_detector_tag:
    type: string
    description: tag of the node problem detector container
@ -741,6 +770,13 @@ resources:
                  "$AUTO_HEALING_CONTROLLER": {get_param: auto_healing_controller}
                  "$MAGNUM_AUTO_HEALER_TAG": {get_param: magnum_auto_healer_tag}
                  "$AUTO_SCALING_ENABLED": {get_param: auto_scaling_enabled}
+                  "$CINDER_CSI_ENABLED": {get_param: cinder_csi_enabled}
+                  "$CINDER_CSI_PLUGIN_TAG": {get_param: cinder_csi_plugin_tag}
+                  "$CSI_ATTACHER_TAG": {get_param: csi_attacher_tag}
+                  "$CSI_PROVISIONER_TAG": {get_param: csi_provisioner_tag}
+                  "$CSI_SNAPSHOTTER_TAG": {get_param: csi_snapshotter_tag}
+                  "$CSI_RESIZER_TAG": {get_param: csi_resizer_tag}
+                  "$CSI_NODE_DRIVER_REGISTRAR_TAG": {get_param: csi_node_driver_registrar_tag}
                  "$DRAINO_TAG": {get_param: draino_tag}
                  "$AUTOSCALER_TAG": {get_param: autoscaler_tag}
                  "$MIN_NODE_COUNT": {get_param: min_node_count}
--- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml
@ -748,6 +748,43 @@ parameters:
    default:
      false

+  cinder_csi_enabled:
+    type: boolean
+    description: >
+      true if the cinder csi feature should be enabled
+    default:
+      false
+
+  cinder_csi_plugin_tag:
+    type: string
+    description: tag of cinder csi plugin
+    default: v1.16.0
+
+  csi_attacher_tag:
+    type: string
+    description: tag of csi attacher
+    default: v2.0.0
+
+  csi_provisioner_tag:
+    type: string
+    description: tag of csi provisioner
+    default: v1.4.0
+
+  csi_snapshotter_tag:
+    type: string
+    description: tag of csi snapshotter
+    default: v1.2.2
+
+  csi_resizer_tag:
+    type: string
+    description: tag of csi resizer
+    default: v0.3.0
+
+  csi_node_driver_registrar_tag:
+    type: string
+    description: tag of csi node driver registrar
+    default: v1.1.0
+
  node_problem_detector_tag:
    type: string
    description: tag of the node problem detector container
@ -1140,6 +1177,13 @@ resources:
          auto_healing_controller: {get_param: auto_healing_controller}
          magnum_auto_healer_tag: {get_param: magnum_auto_healer_tag}
          auto_scaling_enabled: {get_param: auto_scaling_enabled}
+          cinder_csi_enabled: {get_param: cinder_csi_enabled}
+          cinder_csi_plugin_tag: {get_param: cinder_csi_plugin_tag}
+          csi_attacher_tag: {get_param: csi_attacher_tag}
+          csi_provisioner_tag: {get_param: csi_provisioner_tag}
+          csi_snapshotter_tag: {get_param: csi_snapshotter_tag}
+          csi_resizer_tag: {get_param: csi_resizer_tag}
+          csi_node_driver_registrar_tag: {get_param: csi_node_driver_registrar_tag}
          draino_tag: {get_param: draino_tag}
          autoscaler_tag: {get_param: autoscaler_tag}
          min_node_count: {get_param: min_node_count}
@ -1180,6 +1224,7 @@ resources:
                  "$KEYSTONE_AUTH_DEFAULT_POLICY": {get_param: keystone_auth_default_policy}
            - get_file: ../../common/templates/kubernetes/fragments/enable-auto-healing.sh
            - get_file: ../../common/templates/kubernetes/fragments/enable-auto-scaling.sh
+            - get_file: ../../common/templates/kubernetes/fragments/enable-cinder-csi.sh
            # Helm Based Installation Configuration Scripts
            - get_file: ../../common/templates/kubernetes/helm/metrics-server.sh
            - str_replace:
--- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml
@ -529,6 +529,35 @@ parameters:
    description: >
      true if the auto scaling feature should be enabled

+  cinder_csi_enabled:
+    type: boolean
+    description: >
+      true if the cinder csi feature should be enabled
+
+  cinder_csi_plugin_tag:
+    type: string
+    description: tag of cinder csi plugin
+
+  csi_attacher_tag:
+    type: string
+    description: tag of csi attacher
+
+  csi_provisioner_tag:
+    type: string
+    description: tag of csi provisioner
+
+  csi_snapshotter_tag:
+    type: string
+    description: tag of csi snapshotter
+
+  csi_resizer_tag:
+    type: string
+    description: tag of csi resizer
+
+  csi_node_driver_registrar_tag:
+    type: string
+    description: tag of csi node driver registrar
+
  node_problem_detector_tag:
    type: string
    description: tag of the node problem detector container
@ -753,6 +782,13 @@ resources:
                  "$AUTO_HEALING_CONTROLLER": {get_param: auto_healing_controller}
                  "$MAGNUM_AUTO_HEALER_TAG": {get_param: magnum_auto_healer_tag}
                  "$AUTO_SCALING_ENABLED": {get_param: auto_scaling_enabled}
+                  "$CINDER_CSI_ENABLED": {get_param: cinder_csi_enabled}
+                  "$CINDER_CSI_PLUGIN_TAG": {get_param: cinder_csi_plugin_tag}
+                  "$CSI_ATTACHER_TAG": {get_param: csi_attacher_tag}
+                  "$CSI_PROVISIONER_TAG": {get_param: csi_provisioner_tag}
+                  "$CSI_SNAPSHOTTER_TAG": {get_param: csi_snapshotter_tag}
+                  "$CSI_RESIZER_TAG": {get_param: csi_resizer_tag}
+                  "$CSI_NODE_DRIVER_REGISTRAR_TAG": {get_param: csi_node_driver_registrar_tag}
                  "$DRAINO_TAG": {get_param: draino_tag}
                  "$AUTOSCALER_TAG": {get_param: autoscaler_tag}
                  "$MIN_NODE_COUNT": {get_param: min_node_count}
--- a/magnum/tests/unit/drivers/test_template_definition.py
+++ b/magnum/tests/unit/drivers/test_template_definition.py
@ -578,6 +578,20 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
            'magnum_auto_healer_tag')
        auto_scaling_enabled = mock_cluster.labels.get(
            'auto_scaling_enabled')
+        cinder_csi_enabled = mock_cluster.labels.get(
+            'cinder_csi_enabled')
+        cinder_csi_plugin_tag = mock_cluster.labels.get(
+            'cinder_csi_plugin_tag')
+        csi_attacher_tag = mock_cluster.labels.get(
+            'csi_attacher_tag')
+        csi_provisioner_tag = mock_cluster.labels.get(
+            'csi_provisioner_tag')
+        csi_snapshotter_tag = mock_cluster.labels.get(
+            'csi_snapshotter_tag')
+        csi_resizer_tag = mock_cluster.labels.get(
+            'csi_resizer_tag')
+        csi_node_driver_registrar_tag = mock_cluster.labels.get(
+            'csi_node_driver_registrar_tag')
        draino_tag = mock_cluster.labels.get('draino_tag')
        autoscaler_tag = mock_cluster.labels.get('autoscaler_tag')
        min_node_count = mock_cluster.labels.get('min_node_count')
@ -671,6 +685,13 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
            'auto_healing_controller': auto_healing_controller,
            'magnum_auto_healer_tag': magnum_auto_healer_tag,
            'auto_scaling_enabled': auto_scaling_enabled,
+            'cinder_csi_enabled': cinder_csi_enabled,
+            'cinder_csi_plugin_tag': cinder_csi_plugin_tag,
+            'csi_attacher_tag': csi_attacher_tag,
+            'csi_provisioner_tag': csi_provisioner_tag,
+            'csi_snapshotter_tag': csi_snapshotter_tag,
+            'csi_resizer_tag': csi_resizer_tag,
+            'csi_node_driver_registrar_tag': csi_node_driver_registrar_tag,
            'draino_tag': draino_tag,
            'autoscaler_tag': autoscaler_tag,
            'min_node_count': min_node_count,
@ -1047,6 +1068,20 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
            'magnum_auto_healer_tag')
        auto_scaling_enabled = mock_cluster.labels.get(
            'auto_scaling_enabled')
+        cinder_csi_enabled = mock_cluster.labels.get(
+            'cinder_csi_enabled')
+        cinder_csi_plugin_tag = mock_cluster.labels.get(
+            'cinder_csi_plugin_tag')
+        csi_attacher_tag = mock_cluster.labels.get(
+            'csi_attacher_tag')
+        csi_provisioner_tag = mock_cluster.labels.get(
+            'csi_provisioner_tag')
+        csi_snapshotter_tag = mock_cluster.labels.get(
+            'csi_snapshotter_tag')
+        csi_resizer_tag = mock_cluster.labels.get(
+            'csi_resizer_tag')
+        csi_node_driver_registrar_tag = mock_cluster.labels.get(
+            'csi_node_driver_registrar_tag')
        draino_tag = mock_cluster.labels.get('draino_tag')
        autoscaler_tag = mock_cluster.labels.get('autoscaler_tag')
        min_node_count = mock_cluster.labels.get('min_node_count')
@ -1142,6 +1177,13 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
            'auto_healing_controller': auto_healing_controller,
            'magnum_auto_healer_tag': magnum_auto_healer_tag,
            'auto_scaling_enabled': auto_scaling_enabled,
+            'cinder_csi_enabled': cinder_csi_enabled,
+            'cinder_csi_plugin_tag': cinder_csi_plugin_tag,
+            'csi_attacher_tag': csi_attacher_tag,
+            'csi_provisioner_tag': csi_provisioner_tag,
+            'csi_snapshotter_tag': csi_snapshotter_tag,
+            'csi_resizer_tag': csi_resizer_tag,
+            'csi_node_driver_registrar_tag': csi_node_driver_registrar_tag,
            'draino_tag': draino_tag,
            'autoscaler_tag': autoscaler_tag,
            'min_node_count': min_node_count,
--- a/releasenotes/notes/cinder-csi-enabled-label-ab2b8ade63c57cf3.yaml
+++ b/releasenotes/notes/cinder-csi-enabled-label-ab2b8ade63c57cf3.yaml
@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Add cinder_csi_enabled label to support out of tree Cinder CSI.