Decouple Ceph pools creation from sysinv

For a containerized deployment: - the following Ceph pools are no longer created by sysinv: cinder-volumes, images, ephemeral, kube-rbd; the creation has been left/moved to the helm charts as follows: -> the cinder chart: cinder-volumes, cinder.backup -> the rbd-provisioner chart: kube-rbd pools and the ephemeral pool for nova (temporary) -> glance: images - sysinv no longer supports updating the pool quotas - sysinv no longer supports updating the replication for Ceph pools: the replication is updated in the DB through the 'system storage-backend-modify' command, but the chart is applying the change through the helm overrides when the application is (re)applied - sysinv no longer audits the Ceph pools and adjusts the PG num - sysinv no longer generates the Ceph keys for the Ceph pools and the k8s secrets, as these have been moved to the rbd-provisioner chart - upon storage node lock, we determine which are the existing data Ceph pools and deny lock if they are not empty NOTE: There are still parts of the pool management code that will have to be removed once we switch to a containerized deployment only. I've marked that with "TODO(CephPoolsDecouple)" to easily track it. Validation: - install AIO-SX & Storage setups with --kubernetes and -> add Ceph (primary and secondary tier) -> lock/unlock host/storage hosts -> check pools are not created by sysinv -> generate the stx-openstack application tarball -> system application-upload stx-openstack helm-charts-manifest-no-tests.tgz -> system application-apply stx-openstack - install AIO-SX without --kubernetes -> check lock/unlock - install Storage setup without --kubernetes -> check lock/unlock of storage nodes -> check Ceph pools are created -> test quotas can be changed Story: 2002844 Task: 28190 Change-Id: Ic2190e488917bffebcd16daf895dcddd92c6d9c5 Signed-off-by: Irina Mihai <irina.mihai@windriver.com>
2018-11-28 03:41:08 +00:00 · 2018-11-28 03:41:08 +00:00 · 92141980f2
parent ccc36779d9
commit 92141980f2
6 changed files with 151 additions and 78 deletions
--- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/clusterrole.yaml
+++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/clusterrole.yaml
@ -31,4 +31,7 @@ rules:
    resources: ["services"]
    resourceNames: ["kube-dns"]
    verbs: ["list", "get"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "list", "update"]
 {{- end}}
--- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/deployment.yaml
+++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/deployment.yaml
@ -21,15 +21,15 @@ spec:
      labels:
        app: {{ .Values.global.name }}
    spec:
+    {{- if (.Values.global.rbac) or (.Values.global.reuseRbac)}}
+      serviceAccount: {{ .Values.rbac.serviceAccount }}
+    {{- end }}
      containers:
      - name: {{ .Values.global.name }}
-        image: {{ .Values.global.image | quote }}
+        image: {{ .Values.images.tags.rbd_provisioner | quote }}
        env:
        - name: PROVISIONER_NAME
          value: ceph.com/rbd
-{{- if (.Values.global.rbac) or (.Values.global.reuseRbac)}}
-      serviceAccount: {{ .Values.rbac.serviceAccount }}
-{{- end }}
 {{- if .Values.global.nodeSelector }}
      nodeSelector:
 {{ .Values.global.nodeSelector | toYaml | trim | indent 8 }}
--- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/pre-install-check-ceph.yaml
+++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/pre-install-check-ceph.yaml
@ -6,47 +6,47 @@
 #
 */}}

-{{- if .Values.global.doPreInstallVerification }}
+{{- if .Values.global.job_storage_init }}
 {{ $root := . }}
 {{ $defaults := .Values.classdefaults}}
 {{ $mount := "/tmp/mount" }}
-{{- range $classConfig := .Values.classes }}
-kind: ConfigMap
+---
 apiVersion: v1
+kind: ConfigMap
 metadata:
  creationTimestamp: 2016-02-18T19:14:38Z
-  name: config-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+  name: config-{{- $root.Values.global.name }}
  namespace: {{ $root.Values.global.namespace }}
-  annotations:
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "-6"
-    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
 data:
  ceph.conf: |
-    {{ $monitors := or $classConfig.monitors $defaults.monitors }}{{ range $index, $element := $monitors}}
+    {{ $monitors := $defaults.monitors }}{{ range $index, $element := $monitors}}
    [mon.{{- $index }}]
    mon_addr = {{ $element }}
    {{- end }}

  check_ceph.sh: |-
    #!/bin/bash
+
    # Copy from read only mount to Ceph config folder
    cp {{ $mount -}}/ceph.conf /etc/ceph/

-    # Set up admin key in Ceph format
-    CEPH_ADMIN_KEY="/etc/ceph/ceph.client.admin.keyring"
-    if [ ! -z "$CEPH_ADMIN_SECRET" ]; then
-    cat <<EOF > $CEPH_ADMIN_KEY
-    [client.admin]
-          key = $CEPH_ADMIN_SECRET
-    EOF
-    else
-      touch $CEPH_ADMIN_KEY
+    if [ ! -z $CEPH_ADMIN_SECRET ]; then
+      kubectl get secret -n kube-system | grep $CEPH_ADMIN_SECRET
+      ret=$?
+      if [ $ret -ne 0 ]; then
+          msg="Create $CEPH_ADMIN_SECRET secret"
+          echo "$msg"
+          kubectl create secret generic $CEPH_ADMIN_SECRET --type="kubernetes.io/rbd" --from-literal=key= --namespace=$NAMESPACE
+          ret=$?
+          if [ $ret -ne 0 ]; then
+            msg="Error creating secret $CEPH_ADMIN_SECRET, exit"
+            echo "$msg"
+            exit $ret
+          fi
+      fi
    fi

-    # Set up pool key in Ceph format
-    CEPH_USER_KEY=/etc/ceph/ceph.client.{{- $classConfig.userId -}}.keyring
-    echo $CEPH_USER_SECRET > $CEPH_USER_KEY
+    touch /etc/ceph/ceph.client.admin.keyring

    # Check if ceph is accessible
    echo "===================================="
@ -55,52 +55,67 @@ data:
    if [ $ret -ne 0 ]; then
      msg="Error: Ceph cluster is not accessible, check Pod logs for details."
      echo "$msg"
-      echo "$msg" > /dev/termination-log
      exit $ret
    fi

-    # Check if pool exists
-    echo "===================================="
-    ceph osd lspools | grep {{ $classConfig.pool }}
-    ret=$?
-    if [ $ret -ne 0 ]; then
-      msg="Error: Ceph pool {{ $classConfig.pool }} is not accessible, check Pod logs for details."
+    set -ex
+
+    # Get the ruleset from the rule name.
+    ruleset=$(ceph osd crush rule dump $POOL_CRUSH_RULE_NAME | grep "\"ruleset\":" | grep -Eo '[0-9]*')
+    # Make sure the pool exists.
+    ceph osd pool stats $POOL_NAME || ceph osd pool create $POOL_NAME $POOL_CHUNK_SIZE
+    # Set pool configuration.
+    ceph osd pool set $POOL_NAME size $POOL_REPLICATION
+    ceph osd pool set $POOL_NAME crush_rule $ruleset
+
+    if [[ -z $USER_ID && -z $CEPH_USER_SECRET ]]; then
+      msg="No need to create secrets for pool $POOL_NAME"
      echo "$msg"
-      echo "$msg" > /dev/termination-log
-      exit $ret
+      exit 0
    fi

+    KEYRING=$(ceph auth get-or-create client.$USER_ID mon "allow r" osd "allow rwx pool=${POOL_NAME}" | sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p')
+    # Set up pool key in Ceph format
+    CEPH_USER_KEYRING=/etc/ceph/ceph.client.$USER_ID.keyring
+    echo $KEYRING > $CEPH_USER_KEYRING
+    IFS=',' read -a POOL_SECRET_NAMESPACES_ARR <<< "${POOL_SECRET_NAMESPACES}"
+
+    for pool_secret_namespace in "${POOL_SECRET_NAMESPACES_ARR[@]}"
+    do
+      kubectl create secret generic $CEPH_USER_SECRET --type="kubernetes.io/rbd" --from-literal=key=$KEYRING --namespace=$pool_secret_namespace
+    done
+
+    set +ex
+
    # Check if pool is accessible using provided credentials
-    echo "===================================="
-    rbd -p {{ $classConfig.pool }} --user {{ $classConfig.userId }} ls -K $CEPH_USER_KEY
+    echo "====================================="
+    rbd -p $POOL_NAME --user $USER_ID ls -K $CEPH_USER_KEYRING
    ret=$?
    if [ $ret -ne 0 ]; then
-      msg="Error: Ceph pool {{ $classConfig.pool }} is not accessible using \
-    credentials for user {{ $classConfig.userId }}, check Pod logs for details."
+      msg="Error: Ceph pool $POOL_NAME is not accessible using \
+    credentials for user $USER_ID, check Pod logs for details."
      echo "$msg"
-      echo "$msg" > /dev/termination-log
      exit $ret
+    else
+      msg="Pool $POOL_NAME accessible"
+      echo "$msg"
    fi

+    ceph -s
+
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: check-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+  name: rbd-provisioner-storage-init
  namespace: {{ $root.Values.global.namespace }}
  labels:
    heritage: {{$root.Release.Service | quote }}
    release: {{$root.Release.Name | quote }}
    chart: "{{$root.Chart.Name}}-{{$root.Chart.Version}}"
-  annotations:
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
 spec:
-  # Note due to https://github.com/kubernetes/kubernetes/issues/62382
-  # backoffLimit doesn't work in 1.10.x
-  backoffLimit: 1  # Limit the number of job restart in case of failure
-  activeDeadlineSeconds: 60
+  backoffLimit: 3  # Limit the number of job restart in case of failure
+  activeDeadlineSeconds: 180
  template:
    metadata:
      name: "{{$root.Release.Name}}"
@ -110,29 +125,57 @@ spec:
        release: {{$root.Release.Name | quote }}
        chart: "{{$root.Chart.Name}}-{{$root.Chart.Version}}"
    spec:
-      restartPolicy: Never
+      serviceAccountName: {{ $root.Values.rbac.serviceAccount }}
+      restartPolicy: OnFailure
      volumes:
-        - name: config-volume-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+        - name: config-volume-{{- $root.Values.global.name }}
          configMap:
-            name: config-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
+            name: config-{{- $root.Values.global.name }}
      containers:
-        - name: pre-install-job-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
-          image: {{ $root.Values.global.image | quote }}
+        {{- range $classConfig := $root.Values.classes }}
+        - name: storage-init-{{- $classConfig.name }}
+          image: {{ $root.Values.images.tags.rbd_provisioner_storage_init | quote }}
          command: [ "/bin/bash", "{{ $mount }}/check_ceph.sh" ]
-          volumeMounts:
-            - name: config-volume-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
-              mountPath: {{ $mount }}
          env:
+            - name: NAMESPACE
+              value: {{ $root.Values.global.namespace }}
+            - name: POOL_SECRET_NAMESPACES
+              value: {{ $classConfig.pool_secrets_namespaces }}
            - name: CEPH_ADMIN_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: {{ or $classConfig.adminSecretName $defaults.adminSecretName }}
-                  key: key
+              value: {{ $defaults.adminSecretName }}
            - name: CEPH_USER_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: {{ or $classConfig.userSecretName }}
-                  key: key
---
-{{- end }}
+              value: {{ $classConfig.userSecretName }}
+            - name: USER_ID
+              value: {{ $classConfig.userId }}
+            - name: POOL_NAME
+              value: {{ $classConfig.pool_name }}
+            - name: POOL_REPLICATION
+              value: {{ $classConfig.replication | quote }}
+            - name: POOL_CRUSH_RULE_NAME
+              value: {{ $classConfig.crush_rule_name | quote }}
+            - name: POOL_CHUNK_SIZE
+              value: {{ $classConfig.chunk_size | quote }}
+          volumeMounts:
+            - name: config-volume-{{- $root.Values.global.name }}
+              mountPath: {{ $mount }}
+        {{- end }}
+        {{- range $ephemeralPool := $root.Values.ephemeral_pools }}
+        - name: storage-init-{{- $ephemeralPool.pool_name }}
+          image: {{ $root.Values.images.tags.rbd_provisioner_storage_init | quote }}
+          command: [ "/bin/bash", "{{ $mount }}/check_ceph.sh" ]
+          env:
+            - name: NAMESPACE
+              value: {{ $root.Values.global.namespace }}
+            - name: POOL_NAME
+              value: {{ $ephemeralPool.pool_name }}
+            - name: POOL_REPLICATION
+              value: {{ $ephemeralPool.replication | quote }}
+            - name: POOL_CRUSH_RULE_NAME
+              value: {{ $ephemeralPool.crush_rule_name | quote }}
+            - name: POOL_CHUNK_SIZE
+              value: {{ $ephemeralPool.chunk_size | quote }}
+          volumeMounts:
+            - name: config-volume-{{- $root.Values.global.name }}
+              mountPath: {{ $mount }}
+        {{- end }}
 {{- end }}
--- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/role.yaml
+++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/role.yaml
@ -15,5 +15,5 @@ metadata:
 rules:
 - apiGroups: [""]
  resources: ["secrets"]
-  verbs: ["get"]
+  verbs: ["get", "create", "list", "update"]
 {{- end}}
--- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/storageclass.yaml
+++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/storageclass.yaml
@ -20,7 +20,7 @@ parameters:
    adminId: {{ or $classConfig.adminId $defaults.adminId}}
    adminSecretName: {{ or $classConfig.adminSecretName $defaults.adminSecretName }}
    adminSecretNamespace: {{ $namespace }}
-    pool: {{ or $classConfig.pool $defaults.pool }}
+    pool: {{ or $classConfig.pool_name $defaults.pool_name }}
    userId: {{ or $classConfig.userId $defaults.userId }}
    userSecretName: {{ $classConfig.userSecretName }}
    imageFormat: {{ or $classConfig.imageFormat $defaults.imageFormat | quote }}
--- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/values.yaml
+++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/values.yaml
@ -17,14 +17,12 @@ global:
  #
  namespace: kube-system
  #
-  # Run pre-install verifications or skip them.
-  # Skipping them is not recommended
+  # Execute initialization job to verify external Ceph cluster access
+  # and setup additional dependencies assumed by dependent helm charts
+  # (i.e. configmap and secrets).
+  # Skipping is not recommended.
  #
-  doPreInstallVerification: True
-  #
-  # Defines Provisioner's image name including container registry.
-  #
-  image: quay.io/external_storage/rbd-provisioner:latest
+  job_storage_init: true
  #
  # Defines whether to reuse an already defined RBAC policy.
  # Make sure that the serviceAccount defined in the RBAC section matches the one
@ -132,11 +130,40 @@ classdefaults:
 classes:
 - name: fast-rbd # Name of storage class.
  # Ceph pool name
-  pool: kube
+  pool_name: kube
  # Ceph user name to access this pool
  userId: kube
  # K8 secret name with key for accessing the Ceph pool
  userSecretName: ceph-secret-kube
+  # Namespaces for creating the k8s secrets for accessing the Ceph pools
+  pool_secrets_namespaces: kube-system
+  # Name of pool to configure
+  pool_name: kube-rbd
+  # Pool replication
+  replication: 1
+  # Pool crush rule name
+  crush_rule_name: storage_tier_ruleset
+  # Pool chunk size / PG_NUM
+  chunk_size: 8
+# Configuration data for the ephemeral pool(s)
+ephemeral_pools:
+- chunk_size: 8
+  crush_rule_name: storage_tier_ruleset
+  pool_name: ephemeral
+  replication: 1

-
-
+#
+# Defines:
+# - Provisioner's image name including container registry.
+# - CEPH helper image
+#
+images:
+  tags:
+    rbd_provisioner: quay.io/external_storage/rbd-provisioner:latest
+    rbd_provisioner_storage_init: docker.io/port/ceph-config-helper:v1.10.3
+  pull_policy: "IfNotPresent"
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync