From 978507351f1545a71b4c8e3c5180df40f99fb155 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Tue, 5 Dec 2023 13:28:26 -0600 Subject: [PATCH] Add ceph-adapter-rook chart When using Rook for managing Ceph clusters we have to provision a minimal set of assets (keys, endpoints, etc.) to make Openstack-Helm charts work with these Ceph clusters. Rook provides CRDs that can be used for managing Ceph assets like pools/keyrings/buckets etc. but Openstack-Helm can not utilize these CRDs. To support these CRDs in OSH would require having lots of conditionals in OSH templates since we still want OSH to work with OSH ceph-* charts. Change-Id: If7fe29052640e48c37b653e13a74d95e360a6d16 --- ceph-adapter-rook/Chart.yaml | 20 +++ ceph-adapter-rook/README.md | 51 +++++++ ceph-adapter-rook/requirements.yaml | 18 +++ ...amespace-client-ceph-config-manager.sh.tpl | 36 +++++ .../bin/_namespace-client-key-manager.sh.tpl | 51 +++++++ .../bin/_storage-keyring-manager.sh.tpl | 91 ++++++++++++ .../templates/configmap-bin.yaml | 30 ++++ .../templates/configmap-etc-client.yaml | 49 +++++++ .../templates/configmap-templates.yaml | 25 ++++ .../job-namespace-client-ceph-config.yaml | 134 +++++++++++++++++ .../templates/job-namespace-client-key.yaml | 136 ++++++++++++++++++ .../templates/job-storage-admin-keys.yaml | 128 +++++++++++++++++ .../templates/service-mon-discovery.yaml | 37 +++++ ceph-adapter-rook/values.yaml | 119 +++++++++++++++ releasenotes/notes/ceph-adapter-rook.yaml | 4 + tools/deployment/ceph/ceph-adapter-rook.sh | 54 +++++++ tools/deployment/ceph/ceph-rook.sh | 78 ---------- zuul.d/jobs.yaml | 4 +- 18 files changed, 985 insertions(+), 80 deletions(-) create mode 100644 ceph-adapter-rook/Chart.yaml create mode 100644 ceph-adapter-rook/README.md create mode 100644 ceph-adapter-rook/requirements.yaml create mode 100644 ceph-adapter-rook/templates/bin/_namespace-client-ceph-config-manager.sh.tpl create mode 100644 ceph-adapter-rook/templates/bin/_namespace-client-key-manager.sh.tpl create mode 100644 ceph-adapter-rook/templates/bin/_storage-keyring-manager.sh.tpl create mode 100644 ceph-adapter-rook/templates/configmap-bin.yaml create mode 100644 ceph-adapter-rook/templates/configmap-etc-client.yaml create mode 100644 ceph-adapter-rook/templates/configmap-templates.yaml create mode 100644 ceph-adapter-rook/templates/job-namespace-client-ceph-config.yaml create mode 100644 ceph-adapter-rook/templates/job-namespace-client-key.yaml create mode 100644 ceph-adapter-rook/templates/job-storage-admin-keys.yaml create mode 100644 ceph-adapter-rook/templates/service-mon-discovery.yaml create mode 100644 ceph-adapter-rook/values.yaml create mode 100644 releasenotes/notes/ceph-adapter-rook.yaml create mode 100755 tools/deployment/ceph/ceph-adapter-rook.sh diff --git a/ceph-adapter-rook/Chart.yaml b/ceph-adapter-rook/Chart.yaml new file mode 100644 index 000000000..28161a815 --- /dev/null +++ b/ceph-adapter-rook/Chart.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +appVersion: v1.0.0 +description: OpenStack-Helm Ceph Adapter Rook +name: ceph-adapter-rook +version: 0.1.0 +home: https://github.com/ceph/ceph +... diff --git a/ceph-adapter-rook/README.md b/ceph-adapter-rook/README.md new file mode 100644 index 000000000..0a1f457db --- /dev/null +++ b/ceph-adapter-rook/README.md @@ -0,0 +1,51 @@ +# Summary +This is the minimal set of templates necessary to make the rest +of Openstack-Helm charts work with Ceph clusters managed by the +Rook operator. Rook operator not only deploys Ceph clusters but +also provides convenience when interfacing with those clusters +via CRDs which can be used for managing pools/keys/users etc. +However Openstack-Helm charts do not utilize Rook CRDs but instead +manage Ceph assets like pools/keyrings/users/buckets etc. by means +of running bootstrap scripts. Before using Openstack-Helm charts we +have to provision a minimal set of assets like Ceph admin keys and +endpoints and this chart provides exactly this minimal set of templates. + +# Usage +Deploy Ceph admin key and Ceph mon endpoint in the namespace where Ceph cluster is deployed. +``` +tee > /tmp/ceph-adapter-rook-ceph.yaml < /tmp/ceph-adapter-rook-openstack.yaml <= 0.1.0" +... diff --git a/ceph-adapter-rook/templates/bin/_namespace-client-ceph-config-manager.sh.tpl b/ceph-adapter-rook/templates/bin/_namespace-client-ceph-config-manager.sh.tpl new file mode 100644 index 000000000..916e2b9fa --- /dev/null +++ b/ceph-adapter-rook/templates/bin/_namespace-client-ceph-config-manager.sh.tpl @@ -0,0 +1,36 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{- $envAll := . }} + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +# TODO: Get endpoint from rook-ceph-mon-endpoints configmap +ENDPOINT=$(mon_host_from_k8s_ep ${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} ceph-mon-discovery) + +if [ -z "$ENDPOINT" ]; then + echo "Ceph Mon endpoint is empty" + exit 1 +else + echo $ENDPOINT +fi + +kubectl get cm ${CEPH_CONF_ETC} -n ${DEPLOYMENT_NAMESPACE} -o yaml | \ + sed "s#mon_host.*#mon_host = ${ENDPOINT}#g" | \ + kubectl apply -f - + +kubectl get cm ${CEPH_CONF_ETC} -n ${DEPLOYMENT_NAMESPACE} -o yaml diff --git a/ceph-adapter-rook/templates/bin/_namespace-client-key-manager.sh.tpl b/ceph-adapter-rook/templates/bin/_namespace-client-key-manager.sh.tpl new file mode 100644 index 000000000..f0d096450 --- /dev/null +++ b/ceph-adapter-rook/templates/bin/_namespace-client-key-manager.sh.tpl @@ -0,0 +1,51 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{- $envAll := . }} + +CEPH_RBD_KEY=$(kubectl get secret ${PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME} \ + --namespace=${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} \ + -o json ) + +# CONNECT_TO_ROOK_CEPH_CLUSTER is unset by default +if [[ ${CONNECT_TO_ROOK_CEPH_CLUSTER} == "true" ]] ; then + CEPH_CLUSTER_KEY=$(echo "${CEPH_RBD_KEY}" | jq -r '.data["ceph-secret"]') +else + CEPH_CLUSTER_KEY=$(echo "${CEPH_RBD_KEY}" | jq -r '.data.key') +fi + +ceph_activate_namespace() { + kube_namespace=$1 + secret_type=$2 + secret_name=$3 + ceph_key=$4 + { + cat < +create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME} + +function create_kube_storage_key () { + CEPH_KEYRING=$1 + KUBE_SECRET_NAME=$2 + + if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then + { + cat < +create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME} + +{{ else }} + +echo "Not touching ${KUBE_SECRET_NAME} as this is not the initial deployment" + +{{ end }} diff --git a/ceph-adapter-rook/templates/configmap-bin.yaml b/ceph-adapter-rook/templates/configmap-bin.yaml new file mode 100644 index 000000000..235a1a2c4 --- /dev/null +++ b/ceph-adapter-rook/templates/configmap-bin.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} +data: + keys-storage-keyring-manager.sh: | +{{ tuple "bin/_storage-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + provisioner-rbd-namespace-client-key-manager.sh: | +{{ tuple "bin/_namespace-client-key-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + provisioner-rbd-namespace-client-ceph-config-manager.sh: | +{{ tuple "bin/_namespace-client-ceph-config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + +{{- end }} \ No newline at end of file diff --git a/ceph-adapter-rook/templates/configmap-etc-client.yaml b/ceph-adapter-rook/templates/configmap-etc-client.yaml new file mode 100644 index 000000000..c64308ad8 --- /dev/null +++ b/ceph-adapter-rook/templates/configmap-etc-client.yaml @@ -0,0 +1,49 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{/* +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} +{{- end -}} +*/}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} + +{{- end }} +{{- end }} + +{{- if .Values.manifests.configmap_etc }} +{{- list .Values.ceph_configmap_name . | include "ceph.configmap.etc" }} +{{- end }} diff --git a/ceph-adapter-rook/templates/configmap-templates.yaml b/ceph-adapter-rook/templates/configmap-templates.yaml new file mode 100644 index 000000000..92b92a02b --- /dev/null +++ b/ceph-adapter-rook/templates/configmap-templates.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_templates }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }} +data: + admin.keyring: | +{{ .Values.conf.templates.keyring.admin | indent 4 }} +{{- end }} diff --git a/ceph-adapter-rook/templates/job-namespace-client-ceph-config.yaml b/ceph-adapter-rook/templates/job-namespace-client-ceph-config.yaml new file mode 100644 index 000000000..ff60c1e4a --- /dev/null +++ b/ceph-adapter-rook/templates/job-namespace-client-ceph-config.yaml @@ -0,0 +1,134 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_namespace_client_ceph_config }} +{{- $envAll := . }} + +{{- $randStringSuffix := randAlphaNum 5 | lower }} + +{{- $serviceAccountName := print $envAll.Release.Name "-ceph-ns-ceph-config-generator" }} +{{ tuple $envAll "namespace_client_ceph_config_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} + namespace: {{ .Values.admin_secret_namespace }} +rules: + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} + namespace: {{ .Values.admin_secret_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $serviceAccountName }} + labels: +{{ tuple $envAll "ceph" "client-ceph-config-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "client-ceph-config-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "client_ceph_config_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "namespace_client_ceph_config_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-storage-keys-generator +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "client_ceph_config_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CEPH_CONF_ETC + value: {{ .Values.ceph_configmap_name }} + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE + value: {{ .Values.admin_secret_namespace }} + command: + - /tmp/provisioner-rbd-namespace-client-ceph-config-manager.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-provisioners-bin-clients + mountPath: /tmp/provisioner-rbd-namespace-client-ceph-config-manager.sh + subPath: provisioner-rbd-namespace-client-ceph-config-manager.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-provisioners-bin-clients + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 +{{- end }} diff --git a/ceph-adapter-rook/templates/job-namespace-client-key.yaml b/ceph-adapter-rook/templates/job-namespace-client-key.yaml new file mode 100644 index 000000000..a94540fb7 --- /dev/null +++ b/ceph-adapter-rook/templates/job-namespace-client-key.yaml @@ -0,0 +1,136 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_namespace_client_key }} +{{- $envAll := . }} + +{{- $randStringSuffix := randAlphaNum 5 | lower }} + +{{- $serviceAccountName := print $envAll.Release.Name "-ceph-ns-key-generator" }} +{{ tuple $envAll "namespace_client_key_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} + namespace: {{ .Values.admin_secret_namespace }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} + namespace: {{ .Values.admin_secret_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ printf "%s-%s" $serviceAccountName $randStringSuffix }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $serviceAccountName }} + labels: +{{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "client-key-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "client_key_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "namespace_client_key_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-storage-keys-generator +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "client_key_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PVC_CEPH_RBD_STORAGECLASS_USER_SECRET_NAME + value: {{ .Values.secrets.keys.user }} + - name: PVC_CEPH_RBD_STORAGECLASS_ADMIN_SECRET_NAME + value: {{ .Values.secrets.keys.admin }} + - name: PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE + value: {{ .Values.admin_secret_namespace }} + command: + - /tmp/provisioner-rbd-namespace-client-key-manager.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-provisioners-bin-clients + mountPath: /tmp/provisioner-rbd-namespace-client-key-manager.sh + subPath: provisioner-rbd-namespace-client-key-manager.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-provisioners-bin-clients + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 +{{- end }} diff --git a/ceph-adapter-rook/templates/job-storage-admin-keys.yaml b/ceph-adapter-rook/templates/job-storage-admin-keys.yaml new file mode 100644 index 000000000..9fac4580b --- /dev/null +++ b/ceph-adapter-rook/templates/job-storage-admin-keys.yaml @@ -0,0 +1,128 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_storage_admin_keys }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-storage-keys-generator" }} +{{ tuple $envAll "storage_keys_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Values.admin_secret_namespace }} +rules: + - apiGroups: + - "" + resources: + - pods + - pods/exec + - secrets + verbs: + - get + - create + - patch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Values.admin_secret_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-storage-keys-generator + namespace: {{ .Values.admin_secret_namespace }} + labels: +{{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "ceph-storage-keys-generator" "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "storage_keys_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-storage-keys-generator +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "storage_keys_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_GEN_DIR + value: /tmp + - name: CEPH_TEMPLATES_DIR + value: /tmp/templates + - name: CEPH_KEYRING_NAME + value: ceph.client.admin.keyring + - name: CEPH_KEYRING_TEMPLATE + value: admin.keyring + - name: CEPH_KEYRING_ADMIN_NAME + value: {{ .Values.secrets.keyrings.admin }} + - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME + value: {{ .Values.secrets.keys.admin }} + command: + - /tmp/keys-storage-keyring-manager.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/keys-storage-keyring-manager.sh + subPath: keys-storage-keyring-manager.sh + readOnly: true + - name: ceph-templates + mountPath: /tmp/templates + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-mon-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-templates + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }} + defaultMode: 0444 +{{- end }} diff --git a/ceph-adapter-rook/templates/service-mon-discovery.yaml b/ceph-adapter-rook/templates/service-mon-discovery.yaml new file mode 100644 index 000000000..b37d38bea --- /dev/null +++ b/ceph-adapter-rook/templates/service-mon-discovery.yaml @@ -0,0 +1,37 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_mon_discovery }} +{{- $envAll := . }} +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: mon + port: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: mon-msgr2 + port: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: + app: rook-ceph-mon + ceph_daemon_type: mon + clusterIP: None + publishNotReadyAddresses: true +{{- end }} diff --git a/ceph-adapter-rook/values.yaml b/ceph-adapter-rook/values.yaml new file mode 100644 index 000000000..140fe3d41 --- /dev/null +++ b/ceph-adapter-rook/values.yaml @@ -0,0 +1,119 @@ +--- +images: + pull_policy: IfNotPresent + tags: + ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_focal_18.2.0-1-20231013' + dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0' + image_repo_sync: 'docker.io/library/docker:17.07.0' + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + storage_keys_generator: + pod: + runAsUser: 65534 + container: + ceph_storage_keys_generator: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + client_key_generator: + pod: + runAsUser: 99 + container: + ceph_storage_keys_generator: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + dns_policy: "ClusterFirstWithHostNet" + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + resources: + enabled: false + jobs: + secret_provisioning: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + +secrets: + keyrings: + admin: ceph-client-admin-keyring + keys: + admin: pvc-ceph-conf-combined-storageclass + user: pvc-ceph-client-key + +admin_secret_namespace: ceph +ceph_configmap_name: ceph-etc + +conf: + templates: + keyring: + admin: | + [client.admin] + key = {{ key }} + auid = 0 + caps mds = "allow" + caps mon = "allow *" + caps osd = "allow *" + caps mgr = "allow *" + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + objecter_inflight_op_bytes: "1073741824" + objecter_inflight_ops: 10240 + debug_ms: "0/0" + log_file: /dev/stdout + mon_cluster_log_file: /dev/stdout + # TODO: Get mon host from rook-ceph-mon-endpoints configmap + mon_host: "will be discovered" + +endpoints: + cluster_domain_suffix: cluster.local + ceph_mon: + namespace: ceph + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + mon_msgr2: + default: 3300 + +dependencies: + static: + storage_keys_generator: + jobs: null + +manifests: + configmap_bin: true + configmap_templates: true + configmap_etc: true + job_storage_admin_keys: true + job_namespace_client_key: true + job_namespace_client_ceph_config: true + service_mon_discovery: true +... diff --git a/releasenotes/notes/ceph-adapter-rook.yaml b/releasenotes/notes/ceph-adapter-rook.yaml new file mode 100644 index 000000000..94bc37d3a --- /dev/null +++ b/releasenotes/notes/ceph-adapter-rook.yaml @@ -0,0 +1,4 @@ +--- +ceph-adapter-rook: + - 0.1.0 Initial Chart +... diff --git a/tools/deployment/ceph/ceph-adapter-rook.sh b/tools/deployment/ceph/ceph-adapter-rook.sh new file mode 100755 index 000000000..64357bc85 --- /dev/null +++ b/tools/deployment/ceph/ceph-adapter-rook.sh @@ -0,0 +1,54 @@ + +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +make ceph-adapter + +tee > /tmp/ceph-adapter-rook-ceph.yaml < /tmp/ceph-adapter-rook-openstack.yaml < /tmp/s3_admin.yaml <