From 728c340dc0c6a8aa57f43eeee04a524ac2683647 Mon Sep 17 00:00:00 2001 From: "Sigunov, Vladimir (vs422h)" Date: Fri, 10 Dec 2021 15:50:42 -0500 Subject: [PATCH] [CEPH] Discovering ceph-mon endpoints This is a code improvement to reuse ceph monitor doscovering function in different templates. Calling the mentioned above function from a single place (helm-infra snippets) allows less code maintenance and simlifies further development. Rev. 0.1 Charts version bump for ceph-client, ceph-mon, ceph-osd, ceph-provisioners and helm-toolkit Rev. 0.2 Mon endpoint discovery functionality added for the rados gateway. ClusterRole and ClusterRoleBinding added. Rev. 0.3 checkdns is allowed to correct ceph.conf for RGW deployment. Rev. 0.4 Added RoleBinding to the deployment-rgw. Rev. 0.5 Remove _namespace-client-ceph-config-manager.sh.tpl and the appropriate job, because of duplicated functionality. Related configuration has been removed. Rev. 0.6 RoleBinding logic has been changed to meet rules: checkdns namespace - HAS ACCESS -> RGW namespace(s) Change-Id: Ie0af212bdcbbc3aa53335689deed9b226e5d4d89 --- ceph-client/Chart.yaml | 2 +- ...amespace-client-ceph-config-manager.sh.tpl | 36 ------ ceph-client/templates/bin/mds/_start.sh.tpl | 2 +- ceph-client/templates/bin/pool/_init.sh.tpl | 2 +- .../bin/utils/_checkDNS_start.sh.tpl | 51 +++++--- ceph-client/templates/configmap-bin.yaml | 2 - .../templates/deployment-checkdns.yaml | 21 +++- .../templates/job-ns-client-ceph-config.yaml | 114 ------------------ ceph-client/values.yaml | 11 +- ceph-mon/Chart.yaml | 2 +- ceph-mon/templates/bin/mgr/_start.sh.tpl | 2 +- ceph-mon/templates/bin/mon/_start.sh.tpl | 11 +- ceph-mon/templates/bin/moncheck/_start.sh.tpl | 7 +- ceph-osd/Chart.yaml | 2 +- .../bin/osd/ceph-volume/_common.sh.tpl | 9 +- ceph-provisioners/Chart.yaml | 2 +- ...amespace-client-ceph-config-manager.sh.tpl | 6 +- ceph-rgw/Chart.yaml | 2 +- ceph-rgw/templates/bin/rgw/_init.sh.tpl | 26 +++- ceph-rgw/templates/bin/utils/_checkDNS.sh.tpl | 38 ++++++ ceph-rgw/templates/configmap-bin.yaml | 3 +- ceph-rgw/templates/deployment-rgw.yaml | 51 ++++++++ helm-toolkit/Chart.yaml | 2 +- ...e_namespaced_endpoint_namespace_lookup.tpl | 38 ++++++ .../snippets/_mon_host_from_k8s_ep.sh.tpl | 4 +- releasenotes/notes/ceph-client.yaml | 1 + releasenotes/notes/ceph-mon.yaml | 1 + releasenotes/notes/ceph-osd.yaml | 1 + releasenotes/notes/ceph-provisioners.yaml | 1 + releasenotes/notes/ceph-rgw.yaml | 2 + releasenotes/notes/helm-toolkit.yaml | 1 + 31 files changed, 244 insertions(+), 209 deletions(-) delete mode 100644 ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl delete mode 100644 ceph-client/templates/job-ns-client-ceph-config.yaml create mode 100644 ceph-rgw/templates/bin/utils/_checkDNS.sh.tpl create mode 100644 helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl rename {ceph-client => helm-toolkit}/templates/snippets/_mon_host_from_k8s_ep.sh.tpl (94%) diff --git a/ceph-client/Chart.yaml b/ceph-client/Chart.yaml index eb69de660..b79e7e109 100644 --- a/ceph-client/Chart.yaml +++ b/ceph-client/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Client name: ceph-client -version: 0.1.30 +version: 0.1.31 home: https://github.com/ceph/ceph-client ... diff --git a/ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl b/ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl deleted file mode 100644 index 074d9bac1..000000000 --- a/ceph-client/templates/bin/_namespace-client-ceph-config-manager.sh.tpl +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -set -ex -{{- $envAll := . }} - -{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }} - -ENDPOINT=$(mon_host_from_k8s_ep "${DEPLOYMENT_NAMESPACE}" ceph-mon-discovery) - -if [[ -z "${ENDPOINT}" ]]; then - echo "Ceph Mon endpoint is empty" - exit 1 -else - echo "${ENDPOINT}" -fi - -# Update the ceph-client-etc configmap -kubectl get cm "${CEPH_CONF_ETC}" -n "${DEPLOYMENT_NAMESPACE}" -o json | - jq '.data."ceph.conf" |= sub("mon_host = .*";"mon_host = '"${ENDPOINT}"'")' | - kubectl apply -n "${DEPLOYMENT_NAMESPACE}" -f - - -kubectl get cm "${CEPH_CONF_ETC}" -n "${DEPLOYMENT_NAMESPACE}" -o yaml diff --git a/ceph-client/templates/bin/mds/_start.sh.tpl b/ceph-client/templates/bin/mds/_start.sh.tpl index b3fa6604b..15eb4948a 100644 --- a/ceph-client/templates/bin/mds/_start.sh.tpl +++ b/ceph-client/templates/bin/mds/_start.sh.tpl @@ -14,7 +14,7 @@ export LC_ALL=C : "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" -{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }} +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} if [[ ! -e ${CEPH_CONF}.template ]]; then echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" diff --git a/ceph-client/templates/bin/pool/_init.sh.tpl b/ceph-client/templates/bin/pool/_init.sh.tpl index fa55708a6..4563c9be2 100644 --- a/ceph-client/templates/bin/pool/_init.sh.tpl +++ b/ceph-client/templates/bin/pool/_init.sh.tpl @@ -20,7 +20,7 @@ export LC_ALL=C : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" -{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }} +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} if [[ ! -e ${CEPH_CONF}.template ]]; then echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" diff --git a/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl b/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl index 055ab18f5..b4167200f 100644 --- a/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl +++ b/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl @@ -16,33 +16,50 @@ limitations under the License. set -xe -{{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }} +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +{{- $rgwNameSpaces := "" }} +{{- $sep := "" }} +{{- range $_, $ns := .Values.endpoints.ceph_object_store.endpoint_namespaces }} + {{- $rgwNameSpaces = printf "%s%s%s" $rgwNameSpaces $sep $ns }} + {{- $sep = " " }} +{{- end }} + +rgwNameSpaces={{- printf "\"%s\"" $rgwNameSpaces }} function check_mon_dns { - DNS_CHECK=$(getent hosts ceph-mon | head -n1) - PODS=$(kubectl get pods --namespace=${NAMESPACE} --selector=application=ceph --field-selector=status.phase=Running \ - --output=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E 'ceph-mon|ceph-osd|ceph-mgr|ceph-mds') - ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + NS=${1} + # RGWs and the rgw namespace could not exist. Let's check this and prevent this script from failing + if [[ $(kubectl get ns ${NS} -o json | jq -r '.status.phase') == "Active" ]]; then + DNS_CHECK=$(getent hosts ceph-mon | head -n1) + PODS=$(kubectl get pods --namespace=${NS} --selector=application=ceph --field-selector=status.phase=Running \ + --output=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E 'ceph-mon|ceph-osd|ceph-mgr|ceph-mds|ceph-rgw') + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) - if [[ ${PODS} == "" || "${ENDPOINT}" == "" ]]; then - echo "Something went wrong, no PODS or ENDPOINTS are available!" - elif [[ ${DNS_CHECK} == "" ]]; then - for POD in ${PODS}; do - kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \ - sh -c -e "/tmp/utils-checkDNS.sh "${ENDPOINT}"" - done + if [[ ${PODS} == "" || "${ENDPOINT}" == "" ]]; then + echo "Something went wrong, no PODS or ENDPOINTS are available!" + elif [[ ${DNS_CHECK} == "" ]]; then + for POD in ${PODS}; do + kubectl exec -t ${POD} --namespace=${NS} -- \ + sh -c -e "/tmp/utils-checkDNS.sh "${ENDPOINT}"" + done + else + for POD in ${PODS}; do + kubectl exec -t ${POD} --namespace=${NS} -- \ + sh -c -e "/tmp/utils-checkDNS.sh up" + done + fi else - for POD in ${PODS}; do - kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \ - sh -c -e "/tmp/utils-checkDNS.sh up" - done + echo "The namespace ${NS} is not ready, yet" fi } function watch_mon_dns { while [ true ]; do echo "checking DNS health" - check_mon_dns || true + for myNS in ${NAMESPACE} ${rgwNameSpaces}; do + check_mon_dns ${myNS} || true + done echo "sleep 300 sec" sleep 300 done diff --git a/ceph-client/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml index 627947353..04a9987ff 100644 --- a/ceph-client/templates/configmap-bin.yaml +++ b/ceph-client/templates/configmap-bin.yaml @@ -32,8 +32,6 @@ data: init-dirs.sh: | {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - namespace-client-ceph-config-manager.sh: | -{{ tuple "bin/_namespace-client-ceph-config-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} pool-init.sh: | {{ tuple "bin/pool/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph-client/templates/deployment-checkdns.yaml b/ceph-client/templates/deployment-checkdns.yaml index 25b056cea..075f3b8d4 100644 --- a/ceph-client/templates/deployment-checkdns.yaml +++ b/ceph-client/templates/deployment-checkdns.yaml @@ -16,12 +16,19 @@ limitations under the License. {{- $envAll := . }} {{- $serviceAccountName := "ceph-checkdns" }} +{{/* +We will give different name to the RoleBinding resource (see $cephRoleBindingName variable below). +This is neccessary, because the RoleBinding with the default name "ceph-checkdns" exists in the system, +and its reference can not be changed. +*/}} +{{- $cephRoleBindingName := "ceph-checkdns-rolebinding" }} + {{ tuple $envAll "checkdns" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: - name: {{ $serviceAccountName }} + name: clusterrole-checkdns rules: - apiGroups: - "" @@ -29,25 +36,29 @@ rules: - pods - endpoints - pods/exec + - namespaces verbs: - get - list - watch - create --- + apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ $serviceAccountName }} + name: {{ printf "%s-for-%s" $cephRoleBindingName $envAll.Release.Namespace }} + namespace: {{ $envAll.Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $serviceAccountName }} + kind: ClusterRole + name: clusterrole-checkdns subjects: - kind: ServiceAccount name: {{ $serviceAccountName }} namespace: {{ $envAll.Release.Namespace }} --- + kind: Deployment apiVersion: apps/v1 metadata: diff --git a/ceph-client/templates/job-ns-client-ceph-config.yaml b/ceph-client/templates/job-ns-client-ceph-config.yaml deleted file mode 100644 index d1c6a1dcc..000000000 --- a/ceph-client/templates/job-ns-client-ceph-config.yaml +++ /dev/null @@ -1,114 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.manifests.job_ns_client_ceph_config .Values.manifests.configmap_etc }} -{{- $envAll := . }} - -{{- $serviceAccountName := "ceph-ns-client-ceph-config" }} -{{ tuple $envAll "namespace_client_ceph_config_update" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ $serviceAccountName }} -rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - create - - update - - patch - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ $serviceAccountName }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $serviceAccountName }} -subjects: - - kind: ServiceAccount - name: {{ $serviceAccountName }} - namespace: {{ $envAll.Release.Namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ $serviceAccountName }} - labels: -{{ tuple $envAll "ceph" "client-ceph-config-update" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -spec: - template: - metadata: - labels: -{{ tuple $envAll "ceph" "client-ceph-config-update" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: -{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "ceph-client-config-update" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} - spec: -{{ dict "envAll" $envAll "application" "client_ceph_config_update" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} - serviceAccountName: {{ $serviceAccountName }} - restartPolicy: {{ $envAll.Values.jobs.client_ceph_config_update.restartPolicy | quote }} - nodeSelector: - {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} - initContainers: -{{ tuple $envAll "namespace_client_ceph_config_update" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: ceph-client-config-update -{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} -{{ dict "envAll" $envAll "application" "client_ceph_config_update" "container" "ceph_storage_keys_update" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} - env: - - name: CEPH_CONF_ETC - value: "ceph-client-etc" - - name: DEPLOYMENT_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MON_PORT - value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - - name: MON_PORT_V2 - value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - - command: - - /tmp/namespace-client-ceph-config-manager.sh - volumeMounts: - - name: pod-tmp - mountPath: /tmp - - name: pod-etc-ceph - mountPath: /etc/ceph - - name: ceph-client-bin-clients - mountPath: /tmp/namespace-client-ceph-config-manager.sh - subPath: namespace-client-ceph-config-manager.sh - readOnly: true - volumes: - - name: pod-tmp - emptyDir: {} - - name: pod-etc-ceph - emptyDir: {} - - name: ceph-client-bin-clients - configMap: - name: ceph-client-bin - defaultMode: 0555 -{{- end }} diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 8cc40d0aa..4ef64a5f3 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -220,8 +220,6 @@ jobs: startingDeadlineSecs: 60 rbd_pool: restartPolicy: OnFailure - client_ceph_config_update: - restartPolicy: OnFailure conf: features: @@ -540,6 +538,14 @@ endpoints: default: 9283 scheme: default: http + ceph_object_store: + endpoint_namespaces: + - openstack + - ceph + # hosts: + # default: ceph-rgw + # host_fqdn_override: + # default: null manifests: configmap_bin: true @@ -548,7 +554,6 @@ manifests: deployment_mds: true deployment_checkdns: true job_bootstrap: false - job_ns_client_ceph_config: true job_cephfs_client_key: true job_image_repo_sync: true job_rbd_pool: true diff --git a/ceph-mon/Chart.yaml b/ceph-mon/Chart.yaml index c4ccaa836..c22695bd5 100644 --- a/ceph-mon/Chart.yaml +++ b/ceph-mon/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Mon name: ceph-mon -version: 0.1.19 +version: 0.1.20 home: https://github.com/ceph/ceph ... diff --git a/ceph-mon/templates/bin/mgr/_start.sh.tpl b/ceph-mon/templates/bin/mgr/_start.sh.tpl index 4de8de112..d05175cd1 100644 --- a/ceph-mon/templates/bin/mgr/_start.sh.tpl +++ b/ceph-mon/templates/bin/mgr/_start.sh.tpl @@ -6,7 +6,7 @@ set -ex : "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" -{{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }} +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} if [[ ! -e ${CEPH_CONF}.template ]]; then echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" diff --git a/ceph-mon/templates/bin/mon/_start.sh.tpl b/ceph-mon/templates/bin/mon/_start.sh.tpl index b045a39e7..739ac60b3 100644 --- a/ceph-mon/templates/bin/mon/_start.sh.tpl +++ b/ceph-mon/templates/bin/mon/_start.sh.tpl @@ -8,15 +8,16 @@ export LC_ALL=C : "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + if [[ ! -e ${CEPH_CONF}.template ]]; then echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" exit 1 else - ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \ - -v version=v1 -v msgr_version=v2 \ - -v msgr2_port=${MON_PORT_V2} \ - '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',') - if [[ "${ENDPOINT}" == "" ]]; then + + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + + if [[ -z "${ENDPOINT}" ]]; then /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true else /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true diff --git a/ceph-mon/templates/bin/moncheck/_start.sh.tpl b/ceph-mon/templates/bin/moncheck/_start.sh.tpl index 6dd7dfbb6..f1f5fcd08 100644 --- a/ceph-mon/templates/bin/moncheck/_start.sh.tpl +++ b/ceph-mon/templates/bin/moncheck/_start.sh.tpl @@ -3,14 +3,13 @@ set -ex export LC_ALL=C : "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + if [[ ! -e ${CEPH_CONF}.template ]]; then echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" exit 1 else - ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \ - -v version=v1 -v msgr_version=v2 \ - -v msgr2_port=${MON_PORT_V2} \ - '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',') + ENDPOINT=$(mon_host_from_k8s_ep ${NAMESPACE} ceph-mon-discovery) if [[ "${ENDPOINT}" == "" ]]; then /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true else diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml index 263248fa4..f4282deb2 100644 --- a/ceph-osd/Chart.yaml +++ b/ceph-osd/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph OSD name: ceph-osd -version: 0.1.34 +version: 0.1.35 home: https://github.com/ceph/ceph ... diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl index b82f80892..e09ce866e 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl @@ -28,6 +28,8 @@ export PS4='+${BASH_SOURCE:+$(basename ${BASH_SOURCE}):${LINENO}:}${FUNCNAME:+${ : "${OSD_JOURNAL_SIZE:=$(awk '/^osd_journal_size/{print $3}' ${CEPH_CONF}.template)}" : "${OSD_WEIGHT:=1.0}" +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + # Obtain a global lock on /var/lib/ceph/tmp/init-osd.lock function lock() { # Open a file descriptor for the lock file if there isn't one already @@ -136,11 +138,8 @@ if [[ ! -e ${CEPH_CONF}.template ]]; then echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" exit 1 else - ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \ - -v version=v1 -v msgr_version=v2 \ - -v msgr2_port=${MON_PORT_V2} \ - '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',') - if [[ "${ENDPOINT}" == "" ]]; then + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + if [[ -z "${ENDPOINT}" ]]; then /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true else /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true diff --git a/ceph-provisioners/Chart.yaml b/ceph-provisioners/Chart.yaml index 741986d8f..75bc782cc 100644 --- a/ceph-provisioners/Chart.yaml +++ b/ceph-provisioners/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Provisioner name: ceph-provisioners -version: 0.1.16 +version: 0.1.17 home: https://github.com/ceph/ceph ... diff --git a/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-ceph-config-manager.sh.tpl b/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-ceph-config-manager.sh.tpl index 771bb0d62..351bb4d9a 100644 --- a/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-ceph-config-manager.sh.tpl +++ b/ceph-provisioners/templates/bin/provisioner/rbd/_namespace-client-ceph-config-manager.sh.tpl @@ -17,11 +17,9 @@ limitations under the License. set -ex {{- $envAll := . }} +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} -ENDPOINT=$(kubectl get endpoints ceph-mon-discovery -n ${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} -o json | awk -F'"' -v port=${MON_PORT} \ - -v version=v1 -v msgr_version=v2 \ - -v msgr2_port=${MON_PORT_V2} \ - '/"ip"/{print "["version":"$4":"port"/"0","msgr_version":"$4":"msgr2_port"/"0"]"}' | paste -sd',') +ENDPOINT=$(mon_host_from_k8s_ep ${PVC_CEPH_RBD_STORAGECLASS_DEPLOYED_NAMESPACE} ceph-mon-discovery) if [ -z "$ENDPOINT" ]; then echo "Ceph Mon endpoint is empty" diff --git a/ceph-rgw/Chart.yaml b/ceph-rgw/Chart.yaml index d54cfebf1..127908c81 100644 --- a/ceph-rgw/Chart.yaml +++ b/ceph-rgw/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph RadosGW name: ceph-rgw -version: 0.1.17 +version: 0.1.19 home: https://github.com/ceph/ceph ... diff --git a/ceph-rgw/templates/bin/rgw/_init.sh.tpl b/ceph-rgw/templates/bin/rgw/_init.sh.tpl index 66dc03e06..3e6932f72 100644 --- a/ceph-rgw/templates/bin/rgw/_init.sh.tpl +++ b/ceph-rgw/templates/bin/rgw/_init.sh.tpl @@ -15,10 +15,32 @@ limitations under the License. */}} set -ex +export LC_ALL=C -cp -va /etc/ceph/ceph.conf.template /etc/ceph/ceph.conf +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +: "${EP:=ceph-mon-discovery}" +{{- if empty .Values.endpoints.ceph_mon.namespace -}} +MON_NS=ceph +{{ else }} +MON_NS={{ .Values.endpoints.ceph_mon.namespace }} +{{- end }} -cat >> /etc/ceph/ceph.conf <> ${CEPH_CONF} < /dev/null 2>&1 + else + echo "endpoints are already cached in ${CEPH_CONF}" + exit + fi + fi +} + +check_mon_dns + +exit diff --git a/ceph-rgw/templates/configmap-bin.yaml b/ceph-rgw/templates/configmap-bin.yaml index 4a0212780..666cc16dc 100644 --- a/ceph-rgw/templates/configmap-bin.yaml +++ b/ceph-rgw/templates/configmap-bin.yaml @@ -29,7 +29,6 @@ data: bootstrap.sh: | {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} - rgw-restart.sh: | {{ tuple "bin/_rgw-restart.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} init-dirs.sh: | @@ -49,4 +48,6 @@ data: {{ tuple "bin/_create-rgw-placement-targets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} helm-tests.sh: | {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS.sh: | +{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/ceph-rgw/templates/deployment-rgw.yaml b/ceph-rgw/templates/deployment-rgw.yaml index 94c63d35b..cb62515c0 100644 --- a/ceph-rgw/templates/deployment-rgw.yaml +++ b/ceph-rgw/templates/deployment-rgw.yaml @@ -23,6 +23,7 @@ limitations under the License. {{- end }} {{- $serviceAccountName := "ceph-rgw" }} +{{- $checkDnsServiceAccountName := "ceph-checkdns" }} {{- $_ := set $envAll.Values "__depParams" ( list ) }} {{- if .Values.conf.rgw_ks.enabled -}} @@ -37,6 +38,52 @@ limitations under the License. {{- $_ := include "helm-toolkit.utils.dependency_resolver" $dependencyOpts | toString | fromYaml }} {{ tuple $envAll "pod_dependency" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Values.endpoints.ceph_mon.namespace }} +rules: + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Values.endpoints.ceph_mon.namespace }} +roleRef: + kind: Role + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +# This role bindig refers to the ClusterRole for +# check-dns deployment. +# See: openstack-helm-infra/ceph-client/deployment-checkdns.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-from-%s-to-%s" $checkDnsServiceAccountName $envAll.Values.endpoints.ceph_mon.namespace $envAll.Release.Namespace }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clusterrole-checkdns +subjects: + - kind: ServiceAccount + name: {{ $checkDnsServiceAccountName }} + namespace: {{ .Values.endpoints.ceph_mon.namespace }} +--- kind: Deployment apiVersion: apps/v1 metadata: @@ -172,6 +219,10 @@ spec: mountPath: /tmp/rgw-start.sh subPath: rgw-start.sh readOnly: true + - name: ceph-rgw-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true - name: ceph-rgw-etc mountPath: /etc/ceph/ceph.conf.template subPath: ceph.conf diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml index 1df0bb73a..01f56be6d 100644 --- a/helm-toolkit/Chart.yaml +++ b/helm-toolkit/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Helm-Toolkit name: helm-toolkit -version: 0.2.31 +version: 0.2.32 home: https://docs.openstack.org/openstack-helm icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png sources: diff --git a/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl b/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl new file mode 100644 index 000000000..cc4d4de62 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl @@ -0,0 +1,38 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves the namespace scoped hostname for an endpoint +values: | + endpoints: + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: null +usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_namespace_lookup" }} +return: | + default +*/}} + +{{- define "helm-toolkit.endpoints.hostname_namespaced_endpoint_namespace_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- $namespace := $endpointMap.namespace | default $context.Release.Namespace }} +{{- printf "%s" $namespace -}} +{{- end -}} diff --git a/ceph-client/templates/snippets/_mon_host_from_k8s_ep.sh.tpl b/helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl similarity index 94% rename from ceph-client/templates/snippets/_mon_host_from_k8s_ep.sh.tpl rename to helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl index 5b31b3514..fc74c6fb4 100644 --- a/ceph-client/templates/snippets/_mon_host_from_k8s_ep.sh.tpl +++ b/helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl @@ -1,4 +1,4 @@ -{{- define "ceph-client.snippets.mon_host_from_k8s_ep" -}} +{{- define "helm-toolkit.snippets.mon_host_from_k8s_ep" -}} {{/* Inserts a bash function definition mon_host_from_k8s_ep() which can be used @@ -10,7 +10,7 @@ Usage (e.g. in _script.sh.tpl): : "${NS:=ceph}" : "${EP:=ceph-mon-discovery}" - {{ include "ceph-client.snippets.mon_host_from_k8s_ep" . }} + {{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} MON_HOST=$(mon_host_from_k8s_ep "$NS" "$EP") diff --git a/releasenotes/notes/ceph-client.yaml b/releasenotes/notes/ceph-client.yaml index 3cfa9e309..7936f3c4a 100644 --- a/releasenotes/notes/ceph-client.yaml +++ b/releasenotes/notes/ceph-client.yaml @@ -31,4 +31,5 @@ ceph-client: - 0.1.28 Fix ceph.conf update job labels, rendering - 0.1.29 Consolidate mon_host discovery - 0.1.30 Move ceph-mgr deployment to the ceph-mon chart + - 0.1.31 Consolidate mon_endpoints discovery ... diff --git a/releasenotes/notes/ceph-mon.yaml b/releasenotes/notes/ceph-mon.yaml index 202c160b4..3424d6b4c 100644 --- a/releasenotes/notes/ceph-mon.yaml +++ b/releasenotes/notes/ceph-mon.yaml @@ -20,4 +20,5 @@ ceph-mon: - 0.1.17 Skip monmap endpoint check for missing mons - 0.1.18 Move ceph-mgr deployment to the ceph-mon chart - 0.1.19 Add a post-apply job to restart mons after mgrs + - 0.1.20 Consolidate mon_endpoints discovery ... diff --git a/releasenotes/notes/ceph-osd.yaml b/releasenotes/notes/ceph-osd.yaml index f93463d6d..ee494574e 100644 --- a/releasenotes/notes/ceph-osd.yaml +++ b/releasenotes/notes/ceph-osd.yaml @@ -35,4 +35,5 @@ ceph-osd: - 0.1.32 Update htk requirements - 0.1.33 Update log-runner container for MAC - 0.1.34 Remove wait for misplaced objects during OSD restarts + - 0.1.35 Consolidate mon_endpoints discovery ... diff --git a/releasenotes/notes/ceph-provisioners.yaml b/releasenotes/notes/ceph-provisioners.yaml index 66164df78..72dd91d9e 100644 --- a/releasenotes/notes/ceph-provisioners.yaml +++ b/releasenotes/notes/ceph-provisioners.yaml @@ -16,4 +16,5 @@ ceph-provisioners: - 0.1.14 Helm 3 - Fix Job labels - 0.1.15 Add support to connect to rook-ceph cluster - 0.1.16 Update htk requirements + - 0.1.17 Consolidate mon_endpoints discovery ... diff --git a/releasenotes/notes/ceph-rgw.yaml b/releasenotes/notes/ceph-rgw.yaml index 22804496c..29d8cc899 100644 --- a/releasenotes/notes/ceph-rgw.yaml +++ b/releasenotes/notes/ceph-rgw.yaml @@ -18,4 +18,6 @@ ceph-rgw: - 0.1.15 Correct rgw placement target functions - 0.1.16 Helm 3 - Fix Job labels - 0.1.17 Update htk requirements + - 0.1.18 Consolidate mon_endpoints discovery + - 0.1.19 Add ClusterRole to the bootstrap-job ... diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml index b25d5d0d5..62b4a984d 100644 --- a/releasenotes/notes/helm-toolkit.yaml +++ b/releasenotes/notes/helm-toolkit.yaml @@ -38,4 +38,5 @@ helm-toolkit: - 0.2.29 Enhance mariadb backup - 0.2.30 Add ability to image pull secrets on pods - 0.2.31 Add log strings for alert generation + - 0.2.32 Consolidate mon_endpoints discovery ...