Add Ceph pools management chart

- new helm chart to set replication and min replication for each Ceph pool: -> new helm chart name: ceph-pools-audit -> the ceph-pools-audit chart creates a CronJob that runs every 5 minutes; the CronJob checks the replication for each existing pool and sets it right if needed, to reflect the attributes of the Ceph backends -> the CronJob is needed for: charts that may not manage pool configuration, pools created dynamically by services that may not have the current pool configuration uploaded (ex: swift), updating replication without reinstalling the charts that created the pools -> the ceph-pools-audit chart is installed after the rbd-provisioner in the application-apply - new overrides for the ceph-pools-audit chart that provide the replication values from the attributes of the present Ceph backends - enable rados-gw by default when a Ceph backend is enabled Change-Id: I1565268bac3ddc77e8368d2d6ab8600b3e4ed893 Story: 2004520 Task: 29034 Signed-off-by: Irina Mihai <irina.mihai@windriver.com>
2019-01-22 19:32:37 +00:00
parent 1e52662765
commit 754f49a357
17 changed files with 415 additions and 11 deletions
--- a/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data
+++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data
@@ -1,3 +1,3 @@
 SRC_DIR="stx-openstack-helm"
-COPY_LIST_TO_TAR="$PKG_BASE/../../../helm-charts/rbd-provisioner $PKG_BASE/../../../helm-charts/garbd"
-TIS_PATCH_VER=3
+COPY_LIST_TO_TAR="$PKG_BASE/../../../helm-charts/rbd-provisioner $PKG_BASE/../../../helm-charts/garbd $PKG_BASE/../../../helm-charts/ceph-pools-audit"
+TIS_PATCH_VER=4
--- a/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/stx-openstack-helm.spec
+++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/stx-openstack-helm.spec
@@ -58,6 +58,7 @@ helm repo add local http://localhost:8879/charts
 make nova-api-proxy
 make rbd-provisioner
 make garbd
+make ceph-pools-audit

 # terminate helm server (the last backgrounded task)
 kill %1
--- a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml
+++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml
@@ -127,6 +127,35 @@ data:
    - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: openstack-ceph-pools-audit
+data:
+  chart_name: ceph-pools-audit
+  release: openstack-ceph-pools-audit
+  namespace: openstack
+  wait:
+    timeout: 1800
+    labels:
+      app: ceph-pools-audit
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            app: ceph-pools-audit
+  source:
+    type: tar
+    location: http://172.17.0.1/helm_charts/ceph-pools-audit-0.1.0.tgz
+    subpath: ceph-pools-audit
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
  schema: metadata/Document/v1
  name: openstack-mariadb
@@ -2340,6 +2369,16 @@ data:
    - openstack-rbd-provisioner
 ---
 schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ceph-pools-audit
+data:
+  description: "Ceph pools audit"
+  sequenced: false
+  chart_group:
+    - openstack-ceph-pools-audit
+---
+schema: armada/ChartGroup/v1
 metadata:
  schema: metadata/Document/v1
  name: openstack-mariadb
@@ -2456,6 +2495,7 @@ data:
    - kube-system-ingress
    - openstack-ingress
    - provisioner
+    - ceph-pools-audit
    - openstack-mariadb
    - openstack-memcached
    - openstack-rabbitmq
--- a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml
+++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml
@@ -127,6 +127,35 @@ data:
    - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: openstack-ceph-pools-audit
+data:
+  chart_name: ceph-pools-audit
+  release: openstack-ceph-pools-audit
+  namespace: openstack
+  wait:
+    timeout: 1800
+    labels:
+      app: ceph-pools-audit
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            app: ceph-pools-audit
+  source:
+    type: tar
+    location: http://172.17.0.1/helm_charts/ceph-pools-audit-0.1.0.tgz
+    subpath: ceph-pools-audit
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
  schema: metadata/Document/v1
  name: openstack-mariadb
@@ -2340,6 +2369,16 @@ data:
    - openstack-rbd-provisioner
 ---
 schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ceph-pools-audit
+data:
+  description: "Ceph pools audit"
+  sequenced: false
+  chart_group:
+    - openstack-ceph-pools-audit
+---
+schema: armada/ChartGroup/v1
 metadata:
  schema: metadata/Document/v1
  name: openstack-mariadb
@@ -2456,6 +2495,7 @@ data:
    - kube-system-ingress
    - openstack-ingress
    - provisioner
+    - ceph-pools-audit
    - openstack-mariadb
    - openstack-memcached
    - openstack-rabbitmq
--- a/kubernetes/helm-charts/ceph-pools-audit/Chart.yaml
+++ b/kubernetes/helm-charts/ceph-pools-audit/Chart.yaml
@@ -0,0 +1,10 @@
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+appVersion: "1.0"
+description: Ceph RBD pool replication monitor chart
+name: ceph-pools-audit
+version: 0.1.0
--- a/kubernetes/helm-charts/ceph-pools-audit/requirements.yaml
+++ b/kubernetes/helm-charts/ceph-pools-audit/requirements.yaml
@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts
+    version: 0.1.0
--- a/kubernetes/helm-charts/ceph-pools-audit/templates/bin/_ceph-pools-audit.sh.tpl
+++ b/kubernetes/helm-charts/ceph-pools-audit/templates/bin/_ceph-pools-audit.sh.tpl
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+{{/*
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+ceph -s
+ret=$?
+if [ $ret -ne 0 ]; then
+  msg="Error: Ceph cluster is not accessible, check Pod logs for details."
+  echo "$msg"
+  exit $ret
+fi
+
+touch /etc/ceph/ceph.client.admin.keyring
+
+echo "RBD_POOL_CRUSH_RULESET: $RBD_POOL_CRUSH_RULESET"
+if [ -z $RBD_POOL_CRUSH_RULESET ]; then
+    msg="No Ceph crush ruleset specified"
+    echo "$msg"
+    exit 1
+fi
+
+ruleset=$(ceph osd crush rule dump $RBD_POOL_CRUSH_RULESET | grep \"ruleset\" | awk '{print $2}' | grep -Eo '[0-9]+')
+ret=$?
+if [ $ret -ne 0 ]; then
+    msg="Ceph crush ruleset $RBD_POOL_CRUSH_RULESET not found, exit"
+    echo "$msg"
+    exit $ret
+fi
+echo "ruleset: $ruleset"
+
+set -ex
+
+POOLS=( $(ceph osd pool ls) )
+
+for pool_name in "${POOLS[@]}"
+do
+    echo "Check for pool name: $pool_name"
+
+    pool_crush_ruleset=$(ceph osd pool get $pool_name crush_ruleset | awk '{print $2}')
+    echo "pool_crush_ruleset: $pool_crush_ruleset"
+    if [ "$pool_crush_ruleset" != "$ruleset" ]; then
+        continue
+    fi
+
+    pool_size=$(ceph osd pool get $pool_name size | awk '{print $2}')
+    pool_min_size=$(ceph osd pool get $pool_name min_size | awk '{print $2}')
+
+    echo "===> pool_size: $pool_size pool_min_size: $pool_min_size"
+    if [ $pool_size != $RBD_POOL_REPLICATION ]; then
+         echo "set replication for pool $pool_name at $RBD_POOL_REPLICATION"
+         ceph osd pool set $pool_name size $RBD_POOL_REPLICATION
+    fi
+
+    if [ $pool_min_size != $RBD_POOL_MIN_REPLICATION ]; then
+        echo "set min replication for pool $pool_name at $RBD_POOL_MIN_REPLICATION"
+        ceph osd pool set $pool_name min_size $RBD_POOL_MIN_REPLICATION
+    fi
+done
--- a/kubernetes/helm-charts/ceph-pools-audit/templates/confimap-bin.yaml
+++ b/kubernetes/helm-charts/ceph-pools-audit/templates/confimap-bin.yaml
@@ -0,0 +1,19 @@
+{{/*
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.manifests.configmap_bin }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-pools-bin
+data:
+  ceph-pools-audit.sh: |
+{{ tuple "bin/_ceph-pools-audit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
--- a/kubernetes/helm-charts/ceph-pools-audit/templates/job-ceph-pools-audit.yaml
+++ b/kubernetes/helm-charts/ceph-pools-audit/templates/job-ceph-pools-audit.yaml
@@ -0,0 +1,82 @@
+{{/*
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+*/}}
+
+{{- if .Values.manifests.job_ceph_pools_audit }}
+{{- $envAll := . }}
+---
+#
+# The CronJob makes sure all the Ceph pools have the right replication,
+# as present in the attributes of the Ceph backends.
+# This is needed for:
+#   - charts that don't manage pool configuration
+#   - pools created dynamically by services that may not have the current
+#     pool configuration uploaded (ex: swift)
+#   - when replication is changed and we don't want to reinstall all the
+#     charts that created Ceph pools
+#
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: ceph-pools-audit
+spec:
+  schedule: {{ .Values.jobs.job_ceph_pools_audit.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.job_ceph_pools_audit.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.job_ceph_pools_audit.history.failed }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    metadata:
+      name: "{{$envAll.Release.Name}}"
+      namespace: {{ $envAll.Release.namespace }}
+      labels:
+        app: ceph-pools-audit
+    spec:
+      template:
+        metadata:
+          labels:
+            app: ceph-pools-audit
+        spec:
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+          volumes:
+            - name: ceph-pools-bin
+              configMap:
+                name: ceph-pools-bin
+                defaultMode: 0555
+            - name: etcceph
+              emptyDir: {}
+            - name: ceph-etc
+              configMap:
+                name: {{ $envAll.Values.ceph_client.configmap }}
+                defaultMode: 0444
+          containers:
+            {{- range $tierConfig := $envAll.Values.conf.ceph.storage_tiers }}
+            - name: ceph-pools-audit-{{- $tierConfig.name }}
+              image: {{ $envAll.Values.images.tags.ceph_config_helper | quote }}
+              env:
+                - name: RBD_POOL_REPLICATION
+                  value: {{ $tierConfig.replication | quote }}
+                - name: RBD_POOL_MIN_REPLICATION
+                  value: {{ $tierConfig.min_replication | quote }}
+                - name: RBD_POOL_CRUSH_RULESET
+                  value: {{ $tierConfig.crush_ruleset | quote }}
+              command:
+                - /tmp/ceph-pools-audit.sh
+              volumeMounts:
+                - name: ceph-pools-bin
+                  mountPath: /tmp/ceph-pools-audit.sh
+                  subPath: ceph-pools-audit.sh
+                  readOnly: true
+                - name: etcceph
+                  mountPath: /etc/ceph
+                - name: ceph-etc
+                  mountPath: /etc/ceph/ceph.conf
+                  subPath: ceph.conf
+                  readOnly: true
+            {{- end }}
+{{- end }}
--- a/kubernetes/helm-charts/ceph-pools-audit/values.yaml
+++ b/kubernetes/helm-charts/ceph-pools-audit/values.yaml
@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+replicaCount: 1
+
+labels:
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+
+name: ceph-pools-audit
+ceph_client:
+  configmap: ceph-etc
+
+conf:
+  ceph:
+    storage_tiers:
+    - name: ceph-store
+      replication: 2
+      min_replication: 1
+      crush_ruleset: storage_tier_ruleset
+    monitors: []
+
+images:
+  tags:
+    ceph_config_helper: docker.io/port/ceph-config-helper:v1.10.3
+  pullPolicy: "IfNotPresent"
+
+jobs:
+  job_ceph_pools_audit:
+    cron: "*/5 * * * *"
+    history:
+      success: 3
+      failed: 1
+
+resources: {}
+
+nodeSelector: { node-role.kubernetes.io/master: "" }
+
+tolerations: []
+
+affinity: {}
+
+manifests:
+  job_ceph_pools_audit: true
+  configmap_bin: true