Add Ceph pools management chart
- new helm chart to set replication and min replication for
each Ceph pool:
-> new helm chart name: ceph-pools-audit
-> the ceph-pools-audit chart creates a CronJob that runs
every 5 minutes; the CronJob checks the replication for
each existing pool and sets it right if needed, to reflect
the attributes of the Ceph backends
-> the CronJob is needed for: charts that may not manage pool
configuration, pools created dynamically by services that
may not have the current pool configuration uploaded
(ex: swift), updating replication without reinstalling the
charts that created the pools
-> the ceph-pools-audit chart is installed after the
rbd-provisioner in the application-apply
- new overrides for the ceph-pools-audit chart that provide
the replication values from the attributes of the present
Ceph backends
- enable rados-gw by default when a Ceph backend is enabled
Change-Id: I1565268bac3ddc77e8368d2d6ab8600b3e4ed893
Story: 2004520
Task: 29034
Signed-off-by: Irina Mihai <irina.mihai@windriver.com>
This commit is contained in:
Irina Mihai
Al Bailey
parent
6cfef5c237
commit
8b3b2a64ce
|
|
@ -0,0 +1,10 @@
|
|||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
apiVersion: v1
|
||||
appVersion: "1.0"
|
||||
description: Ceph RBD pool replication monitor chart
|
||||
name: ceph-pools-audit
|
||||
version: 0.1.0
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
dependencies:
|
||||
- name: helm-toolkit
|
||||
repository: http://localhost:8879/charts
|
||||
version: 0.1.0
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
#!/bin/bash
|
||||
|
||||
{{/*
|
||||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
*/}}
|
||||
|
||||
ceph -s
|
||||
ret=$?
|
||||
if [ $ret -ne 0 ]; then
|
||||
msg="Error: Ceph cluster is not accessible, check Pod logs for details."
|
||||
echo "$msg"
|
||||
exit $ret
|
||||
fi
|
||||
|
||||
touch /etc/ceph/ceph.client.admin.keyring
|
||||
|
||||
echo "RBD_POOL_CRUSH_RULESET: $RBD_POOL_CRUSH_RULESET"
|
||||
if [ -z $RBD_POOL_CRUSH_RULESET ]; then
|
||||
msg="No Ceph crush ruleset specified"
|
||||
echo "$msg"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ruleset=$(ceph osd crush rule dump $RBD_POOL_CRUSH_RULESET | grep \"ruleset\" | awk '{print $2}' | grep -Eo '[0-9]+')
|
||||
ret=$?
|
||||
if [ $ret -ne 0 ]; then
|
||||
msg="Ceph crush ruleset $RBD_POOL_CRUSH_RULESET not found, exit"
|
||||
echo "$msg"
|
||||
exit $ret
|
||||
fi
|
||||
echo "ruleset: $ruleset"
|
||||
|
||||
set -ex
|
||||
|
||||
POOLS=( $(ceph osd pool ls) )
|
||||
|
||||
for pool_name in "${POOLS[@]}"
|
||||
do
|
||||
echo "Check for pool name: $pool_name"
|
||||
|
||||
pool_crush_ruleset=$(ceph osd pool get $pool_name crush_ruleset | awk '{print $2}')
|
||||
echo "pool_crush_ruleset: $pool_crush_ruleset"
|
||||
if [ "$pool_crush_ruleset" != "$ruleset" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
pool_size=$(ceph osd pool get $pool_name size | awk '{print $2}')
|
||||
pool_min_size=$(ceph osd pool get $pool_name min_size | awk '{print $2}')
|
||||
|
||||
echo "===> pool_size: $pool_size pool_min_size: $pool_min_size"
|
||||
if [ $pool_size != $RBD_POOL_REPLICATION ]; then
|
||||
echo "set replication for pool $pool_name at $RBD_POOL_REPLICATION"
|
||||
ceph osd pool set $pool_name size $RBD_POOL_REPLICATION
|
||||
fi
|
||||
|
||||
if [ $pool_min_size != $RBD_POOL_MIN_REPLICATION ]; then
|
||||
echo "set min replication for pool $pool_name at $RBD_POOL_MIN_REPLICATION"
|
||||
ceph osd pool set $pool_name min_size $RBD_POOL_MIN_REPLICATION
|
||||
fi
|
||||
done
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
{{/*
|
||||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
*/}}
|
||||
|
||||
{{- if .Values.manifests.configmap_bin }}
|
||||
{{- $envAll := . }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: ceph-pools-bin
|
||||
data:
|
||||
ceph-pools-audit.sh: |
|
||||
{{ tuple "bin/_ceph-pools-audit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
{{/*
|
||||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
*/}}
|
||||
|
||||
{{- if .Values.manifests.job_ceph_pools_audit }}
|
||||
{{- $envAll := . }}
|
||||
---
|
||||
#
|
||||
# The CronJob makes sure all the Ceph pools have the right replication,
|
||||
# as present in the attributes of the Ceph backends.
|
||||
# This is needed for:
|
||||
# - charts that don't manage pool configuration
|
||||
# - pools created dynamically by services that may not have the current
|
||||
# pool configuration uploaded (ex: swift)
|
||||
# - when replication is changed and we don't want to reinstall all the
|
||||
# charts that created Ceph pools
|
||||
#
|
||||
apiVersion: batch/v1beta1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: ceph-pools-audit
|
||||
spec:
|
||||
schedule: {{ .Values.jobs.job_ceph_pools_audit.cron | quote }}
|
||||
successfulJobsHistoryLimit: {{ .Values.jobs.job_ceph_pools_audit.history.success }}
|
||||
failedJobsHistoryLimit: {{ .Values.jobs.job_ceph_pools_audit.history.failed }}
|
||||
concurrencyPolicy: Forbid
|
||||
jobTemplate:
|
||||
metadata:
|
||||
name: "{{$envAll.Release.Name}}"
|
||||
namespace: {{ $envAll.Release.namespace }}
|
||||
labels:
|
||||
app: ceph-pools-audit
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ceph-pools-audit
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
nodeSelector:
|
||||
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
|
||||
volumes:
|
||||
- name: ceph-pools-bin
|
||||
configMap:
|
||||
name: ceph-pools-bin
|
||||
defaultMode: 0555
|
||||
- name: etcceph
|
||||
emptyDir: {}
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: {{ $envAll.Values.ceph_client.configmap }}
|
||||
defaultMode: 0444
|
||||
containers:
|
||||
{{- range $tierConfig := $envAll.Values.conf.ceph.storage_tiers }}
|
||||
- name: ceph-pools-audit-{{- $tierConfig.name }}
|
||||
image: {{ $envAll.Values.images.tags.ceph_config_helper | quote }}
|
||||
env:
|
||||
- name: RBD_POOL_REPLICATION
|
||||
value: {{ $tierConfig.replication | quote }}
|
||||
- name: RBD_POOL_MIN_REPLICATION
|
||||
value: {{ $tierConfig.min_replication | quote }}
|
||||
- name: RBD_POOL_CRUSH_RULESET
|
||||
value: {{ $tierConfig.crush_ruleset | quote }}
|
||||
command:
|
||||
- /tmp/ceph-pools-audit.sh
|
||||
volumeMounts:
|
||||
- name: ceph-pools-bin
|
||||
mountPath: /tmp/ceph-pools-audit.sh
|
||||
subPath: ceph-pools-audit.sh
|
||||
readOnly: true
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
#
|
||||
# Copyright (c) 2019 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
labels:
|
||||
job:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
|
||||
name: ceph-pools-audit
|
||||
ceph_client:
|
||||
configmap: ceph-etc
|
||||
|
||||
conf:
|
||||
ceph:
|
||||
storage_tiers:
|
||||
- name: ceph-store
|
||||
replication: 2
|
||||
min_replication: 1
|
||||
crush_ruleset: storage_tier_ruleset
|
||||
monitors: []
|
||||
|
||||
images:
|
||||
tags:
|
||||
ceph_config_helper: docker.io/port/ceph-config-helper:v1.10.3
|
||||
pullPolicy: "IfNotPresent"
|
||||
|
||||
jobs:
|
||||
job_ceph_pools_audit:
|
||||
cron: "*/5 * * * *"
|
||||
history:
|
||||
success: 3
|
||||
failed: 1
|
||||
|
||||
resources: {}
|
||||
|
||||
nodeSelector: { node-role.kubernetes.io/master: "" }
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
|
||||
manifests:
|
||||
job_ceph_pools_audit: true
|
||||
configmap_bin: true
|
||||
Loading…
Reference in New Issue