Add rbd-provisioner to common StarlingX charts

Add the custom StarlingX rbd-provisioner chart to the common custom chart area of the stx-config repo. Add the chart to the stx-openstack application RPM. This chart is based on the content of the ceph rbd storage code (kubernetes-incubator/external-storage): - c463bd18 -> kubernetes-1.12.0-beta.1 Change-Id: Ib5fe40ed82bf5ffdd2eea4ebc1cd534e1b5ceacd Story: 2004005 Task: 27802 Signed-off-by: Robert Church <robert.church@windriver.com>
2018-11-05 04:03:15 -05:00 · 2018-11-05 04:03:15 -05:00 · 7e1d73f95a
commit 7e1d73f95a
parent cb6b0fbde2
16 changed files with 567 additions and 1 deletions
--- a/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data
+++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data
@ -1,2 +1,3 @@
 SRC_DIR="stx-openstack-helm"
+COPY_LIST_TO_TAR="$PKG_BASE/../../../helm-charts/rbd-provisioner"
 TIS_PATCH_VER=1
--- a/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/stx-openstack-helm.spec
+++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/stx-openstack-helm.spec
@ -36,6 +36,7 @@ helm repo add local http://localhost:8879/charts

 # Make the charts. These produce a tgz file
 make nova-api-proxy
+make rbd-provisioner

 # remove helm-toolkit. This will be packaged with openstack-helm-infra
 rm  ./helm-toolkit-%{toolkit_version}.tgz
--- a/kubernetes/helm-charts/placeholder/placeholder.txt
+++ b/kubernetes/helm-charts/placeholder/placeholder.txt
@ -1 +0,0 @@
-This is a placeholder, remove when there are actual helm charts.
--- a/kubernetes/helm-charts/rbd-provisioner/Chart.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/Chart.yaml
@ -0,0 +1,10 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+apiVersion: v1
+description: rbd provisioner chart
+name: rbd-provisioner
+version: 0.1.0
--- a/kubernetes/helm-charts/rbd-provisioner/examples/multi-pool-storage.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/examples/multi-pool-storage.yaml
@ -0,0 +1,22 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+classdefaults:
+  adminId: admin
+  adminSecretName: ceph-admin
+  monitors:
+  - 192.168.204.4:6789
+  - 192.168.204.3:6789
+  - 192.168.204.22:6789
+classes:
+- name: rbd
+  pool: kube-rbd
+  userId: ceph-pool-kube-rbd
+  userSecretName: ceph-pool-kube-rbd
+- name: gold-rbd
+  pool: kube-rbd-gold
+  userId: ceph-pool-gold-kube-rbd-gold
+  userSecretName: ceph-pool-gold-kube-rbd-gold
--- a/kubernetes/helm-charts/rbd-provisioner/examples/single-pool-storage.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/examples/single-pool-storage.yaml
@ -0,0 +1,17 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+classes:
+- name: slow-rbd
+  monitors:
+    - 192.168.204.3:6789
+    - 192.168.204.150:6789
+    - 192.168.204.4:6789
+  adminId: admin
+  adminSecretName: ceph-secret
+  pool: kube
+  userId: kube
+  userSecretName: ceph-secret-kube
--- a/kubernetes/helm-charts/rbd-provisioner/templates/clusterrole.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/clusterrole.yaml
@ -0,0 +1,34 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.global.rbac }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.rbac.clusterRole }}
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["services"]
+    resourceNames: ["kube-dns"]
+    verbs: ["list", "get"]
+{{- end}}
--- a/kubernetes/helm-charts/rbd-provisioner/templates/clusterrolebinding.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/clusterrolebinding.yaml
@ -0,0 +1,22 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.global.rbac }}
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Values.rbac.clusterRoleBinding }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.rbac.serviceAccount }}
+    namespace: {{ .Values.global.namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ .Values.rbac.clusterRole }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end}}
--- a/kubernetes/helm-charts/rbd-provisioner/templates/daemonset.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/daemonset.yaml
@ -0,0 +1,48 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if eq .Values.global.deployAs "DaemonSet" }}
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: {{ .Values.global.name }}
+  namespace: {{ .Values.global.namespace }}
+  labels:
+    app: {{ .Values.global.name }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Values.global.name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.global.name }}
+    spec:
+{{- if (.Values.global.rbac) or (.Values.global.reuseRbac)}}
+      serviceAccountName: {{.Values.rbac.serviceAccount}}
+{{- end}}
+{{- if .Values.global.tolerations }}
+      tolerations:
+{{ .Values.global.tolerations | toYaml | trim | indent 8 }}
+{{- end }}
+{{- if .Values.global.nodeSelector }}
+      nodeSelector:
+{{ .Values.global.nodeSelector | toYaml | trim | indent 8 }}
+{{- end }}
+      containers:
+        - image: {{ .Values.global.image | quote }}
+          name: {{ .Values.global.name }}
+{{- if .Values.global.resources }}
+          resources:
+{{ .Values.global.resources | toYaml | trim | indent 12 }}
+{{- end }}
+          env:
+          - name: PROVISIONER_NAME
+            value: ceph.com/rbd
+{{- end}}
+
--- a/kubernetes/helm-charts/rbd-provisioner/templates/deployment.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/deployment.yaml
@ -0,0 +1,45 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if eq .Values.global.deployAs "Deployment" }}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ .Values.global.name }}
+  namespace: {{ .Values.global.namespace }}
+spec:
+  replicas: {{ .Values.global.replicas }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.global.name }}
+    spec:
+      containers:
+      - name: {{ .Values.global.name }}
+        image: {{ .Values.global.image | quote }}
+        env:
+        - name: PROVISIONER_NAME
+          value: ceph.com/rbd
+{{- if (.Values.global.rbac) or (.Values.global.reuseRbac)}}
+      serviceAccount: {{ .Values.rbac.serviceAccount }}
+{{- end }}
+{{- if .Values.global.nodeSelector }}
+      nodeSelector:
+{{ .Values.global.nodeSelector | toYaml | trim | indent 8 }}
+{{- end }}
+{{- if .Values.global.tolerations }}
+      tolerations:
+{{ .Values.global.tolerations | toYaml | trim | indent 8 }}
+{{- end}}
+{{- if .Values.global.resources }}
+      resources:
+{{ .Values.global.resources | toYaml | trim | indent 8 }}
+{{- end }}
+{{- end }}
--- a/kubernetes/helm-charts/rbd-provisioner/templates/pre-install-check-ceph.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/pre-install-check-ceph.yaml
@ -0,0 +1,138 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.global.doPreInstallVerification }}
+{{ $root := . }}
+{{ $defaults := .Values.classdefaults}}
+{{ $mount := "/tmp/mount" }}
+{{- range $classConfig := .Values.classes }}
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  creationTimestamp: 2016-02-18T19:14:38Z
+  name: config-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+  namespace: {{ $root.Values.global.namespace }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "-6"
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
+data:
+  ceph.conf: |
+    {{ $monitors := or $classConfig.monitors $defaults.monitors }}{{ range $index, $element := $monitors}}
+    [mon.{{- $index }}]
+    mon_addr = {{ $element }}
+    {{- end }}
+
+  check_ceph.sh: |-
+    #!/bin/bash
+    # Copy from read only mount to Ceph config folder
+    cp {{ $mount -}}/ceph.conf /etc/ceph/
+
+    # Set up admin key in Ceph format
+    CEPH_ADMIN_KEY="/etc/ceph/ceph.client.admin.keyring"
+    if [ ! -z "$CEPH_ADMIN_SECRET" ]; then
+    cat <<EOF > $CEPH_ADMIN_KEY
+    [client.admin]
+          key = $CEPH_ADMIN_SECRET
+    EOF
+    else
+      touch $CEPH_ADMIN_KEY
+    fi
+
+    # Set up pool key in Ceph format
+    CEPH_USER_KEY=/etc/ceph/ceph.client.{{- $classConfig.userId -}}.keyring
+    echo $CEPH_USER_SECRET > $CEPH_USER_KEY
+
+    # Check if ceph is accessible
+    echo "===================================="
+    ceph -s
+    ret=$?
+    if [ $ret -ne 0 ]; then
+      msg="Error: Ceph cluster is not accessible, check Pod logs for details."
+      echo "$msg"
+      echo "$msg" > /dev/termination-log
+      exit $ret
+    fi
+
+    # Check if pool exists
+    echo "===================================="
+    ceph osd lspools | grep {{ $classConfig.pool }}
+    ret=$?
+    if [ $ret -ne 0 ]; then
+      msg="Error: Ceph pool {{ $classConfig.pool }} is not accessible, check Pod logs for details."
+      echo "$msg"
+      echo "$msg" > /dev/termination-log
+      exit $ret
+    fi
+
+    # Check if pool is accessible using provided credentials
+    echo "===================================="
+    rbd -p {{ $classConfig.pool }} --user {{ $classConfig.userId }} ls -K $CEPH_USER_KEY
+    ret=$?
+    if [ $ret -ne 0 ]; then
+      msg="Error: Ceph pool {{ $classConfig.pool }} is not accessible using \
+    credentials for user {{ $classConfig.userId }}, check Pod logs for details."
+      echo "$msg"
+      echo "$msg" > /dev/termination-log
+      exit $ret
+    fi
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: check-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+  namespace: {{ $root.Values.global.namespace }}
+  labels:
+    heritage: {{$root.Release.Service | quote }}
+    release: {{$root.Release.Name | quote }}
+    chart: "{{$root.Chart.Name}}-{{$root.Chart.Version}}"
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
+spec:
+  # Note due to https://github.com/kubernetes/kubernetes/issues/62382
+  # backoffLimit doesn't work in 1.10.x
+  backoffLimit: 1  # Limit the number of job restart in case of failure
+  activeDeadlineSeconds: 60
+  template:
+    metadata:
+      name: "{{$root.Release.Name}}"
+      namespace: {{ $root.Values.global.namespace }}
+      labels:
+        heritage: {{$root.Release.Service | quote }}
+        release: {{$root.Release.Name | quote }}
+        chart: "{{$root.Chart.Name}}-{{$root.Chart.Version}}"
+    spec:
+      restartPolicy: Never
+      volumes:
+        - name: config-volume-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+          configMap:
+            name: config-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+      containers:
+        - name: pre-install-job-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+          image: {{ $root.Values.global.image | quote }}
+          command: [ "/bin/bash", "{{ $mount }}/check_ceph.sh" ]
+          volumeMounts:
+            - name: config-volume-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+              mountPath: {{ $mount }}
+          env:
+            - name: CEPH_ADMIN_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ or $classConfig.adminSecretName $defaults.adminSecretName }}
+                  key: key
+            - name: CEPH_USER_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ or $classConfig.userSecretName }}
+                  key: key
+---
+{{- end }}
+{{- end }}
--- a/kubernetes/helm-charts/rbd-provisioner/templates/role.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/role.yaml
@ -0,0 +1,19 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.global.rbac }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Values.rbac.role }}
+  namespace: {{ .Values.global.namespace }}
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get"]
+{{- end}}
--- a/kubernetes/helm-charts/rbd-provisioner/templates/rolebinding.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/rolebinding.yaml
@ -0,0 +1,23 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.global.rbac }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Values.rbac.roleBinding }}
+  namespace: {{ .Values.global.namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Values.rbac.role }}
+subjects:
+- kind: ServiceAccount
+  name: {{ .Values.rbac.serviceAccount }}
+  namespace: {{ .Values.global.namespace }}
+{{- end}}
--- a/kubernetes/helm-charts/rbd-provisioner/templates/serviceaccount.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/serviceaccount.yaml
@ -0,0 +1,15 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.global.rbac }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.rbac.serviceAccount }}
+  namespace: {{ .Values.global.namespace }}
+{{- end }}
--- a/kubernetes/helm-charts/rbd-provisioner/templates/storageclass.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/templates/storageclass.yaml
@ -0,0 +1,30 @@
+{{/*
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.global.provisionStorageClass }}
+{{ $namespace := .Values.global.namespace }}
+{{ $defaults := .Values.classdefaults}}
+{{- range $classConfig := .Values.classes }}
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+    name: {{ $classConfig.name }}
+provisioner: ceph.com/rbd
+parameters:
+    monitors: {{ $monitors := or $classConfig.monitors $defaults.monitors }}{{ join "," $monitors}}
+    adminId: {{ or $classConfig.adminId $defaults.adminId}}
+    adminSecretName: {{ or $classConfig.adminSecretName $defaults.adminSecretName }}
+    adminSecretNamespace: {{ $namespace }}
+    pool: {{ or $classConfig.pool $defaults.pool }}
+    userId: {{ or $classConfig.userId $defaults.userId }}
+    userSecretName: {{ $classConfig.userSecretName }}
+    imageFormat: {{ or $classConfig.imageFormat $defaults.imageFormat | quote }}
+    imageFeatures: {{ or $classConfig.imageFeatures $defaults.imageFeatures}}
+---
+{{- end }}
+{{- end }}
--- a/kubernetes/helm-charts/rbd-provisioner/values.yaml
+++ b/kubernetes/helm-charts/rbd-provisioner/values.yaml
@ -0,0 +1,142 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+#
+# Global options.
+# Defaults should be fine in most cases.
+global:
+  #
+  # Defines the name of a Provisioner.
+  #
+  name: "rbd-provisioner"
+  #
+  # Defines the namespace where provisioner runs.
+  #
+  namespace: kube-system
+  #
+  # Run pre-install verifications or skip them.
+  # Skipping them is not recommended
+  #
+  doPreInstallVerification: True
+  #
+  # Defines Provisioner's image name including container registry.
+  #
+  image: quay.io/external_storage/rbd-provisioner:latest
+  #
+  # Defines whether to reuse an already defined RBAC policy.
+  # Make sure that the serviceAccount defined in the RBAC section matches the one
+  # in the policy you reuse.
+  #
+  reuseRbac: false
+  #
+  # Defines whether to generate service account and role bindings.
+  #
+  rbac: true
+  #
+  # Provision storage class. If false you have to provision storage classes by hand.
+  #
+  provisionStorageClass: true
+  #
+  # Choose if rbd-provisioner pod should be deployed as deplyment or DaemonSet
+  # Values: none, Deployment, DaemonSet
+  #
+  deployAs: Deployment
+  #
+  # If configured, tolerations will add a toleration field to the Pod.
+  #
+  # Node tolerations for rbd-volume-provisioner scheduling to nodes with taints.
+  # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+  # Example:
+  #  [
+  #    {
+  #      "key": "node-role.kubernetes.io/master",
+  #      "operator": "Exists"
+  #    }
+  #  ]
+  #
+  tolerations: []
+  # If configured, resources will set the requests/limits field to the Pod.
+  # Ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
+  # Example:
+  #  {
+  #    "limits": {
+  #      "memory": "200Mi"
+  #    },
+  #    "requests": {
+  #      "cpu": "100m",
+  #      "memory": "200Mi"
+  #    }
+  #  }
+  resources: {}
+  #
+  # Number of replicas to start when configured as deployment
+  #
+  replicas: 1
+  #
+  # Node Selector
+  #
+  nodeSelector: { node-role.kubernetes.io/master: "" }
+#
+# RBAC options.
+# Defaults should be fine in most cases.
+rbac:
+  #
+  # Cluster Role name
+  #
+  clusterRole: rbd-provisioner
+  #
+  # Cluster Role Binding name
+  #
+  clusterRoleBinding: rbd-provisioner
+  #
+  # Role name
+  #
+  role: rbd-provisioner
+  #
+  # Role Binding name
+  #
+  roleBinding: rbd-provisioner
+  #
+  # Defines a name of the service account which Provisioner will use to communicate with API server.
+  #
+  serviceAccount: rbd-provisioner
+
+#
+# Configure storage classes.
+# Defaults for storage classes. Update this if you have a single Ceph storage cluster.
+# No need to add them to each class.
+#
+classdefaults:
+  # Define ip addresses of Ceph Monitors
+  monitors:
+    - 192.168.204.3:6789
+    - 192.168.204.150:6789
+    - 192.168.204.4:6789
+  # Ceph admin account
+  adminId: admin
+  # K8 secret name for the admin context
+  adminSecretName: ceph-secret
+  # Ceph RBD image format version
+  imageFormat: 2
+  # Ceph RBD image features.
+  imageFeatures: layering
+
+#
+# Configure storage classes.
+# This section should be tailored to your setup. It allows you to define multiple storage
+# classes for the same cluster (e.g. if you have tiers of drives with different speeds).
+# If you have multiple Ceph clusters take attributes from classdefaults and add them here.
+classes:
+- name: fast-rbd # Name of storage class.
+  # Ceph pool name
+  pool: kube
+  # Ceph user name to access this pool
+  userId: kube
+  # K8 secret name with key for accessing the Ceph pool
+  userSecretName: ceph-secret-kube
+
+
+
				`@ -1 +0,0 @@`
				`This is a placeholder, remove when there are actual helm charts.`