From d554d74bf0c6d61d63f92e1d8224881dd4fa726d Mon Sep 17 00:00:00 2001 From: okozachenko Date: Fri, 2 Sep 2022 18:50:40 +1000 Subject: [PATCH] Fix cinder host name Now `cinder-volume-worker` is set as DEFAULT.host. It results only one host per volume service type registered regardless of replica numbers. To fix this, removed DEFAULT.host from default values so hostname of pods will be used as host name. It will produce garbage services with down state because pod name changes every creation. To avoid this, added a cronjob for service clean. Change-Id: I9ec6f62105124f9088afdb231099b532fc83bb34 --- cinder/Chart.yaml | 2 +- .../bin/_cinder-service-cleaner.sh.tpl | 31 +++++++ cinder/templates/configmap-bin.yaml | 2 + .../templates/cron-job-service-cleaner.yaml | 85 +++++++++++++++++++ cinder/values.yaml | 37 +++++++- .../values_overrides/train-ubuntu_bionic.yaml | 1 + .../ussuri-ubuntu_bionic.yaml | 1 + .../victoria-ubuntu_focal.yaml | 1 + .../wallaby-ubuntu_focal.yaml | 1 + .../values_overrides/xena-ubuntu_focal.yaml | 1 + .../values_overrides/yoga-ubuntu_focal.yaml | 1 + releasenotes/notes/cinder.yaml | 1 + 12 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 cinder/templates/bin/_cinder-service-cleaner.sh.tpl create mode 100644 cinder/templates/cron-job-service-cleaner.yaml diff --git a/cinder/Chart.yaml b/cinder/Chart.yaml index d235da61c5..77d4ea1fba 100644 --- a/cinder/Chart.yaml +++ b/cinder/Chart.yaml @@ -14,7 +14,7 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Cinder name: cinder -version: 0.2.30 +version: 0.2.31 home: https://docs.openstack.org/cinder/latest/ icon: https://www.openstack.org/themes/openstack/images/project-mascots/Cinder/OpenStack_Project_Cinder_vertical.png sources: diff --git a/cinder/templates/bin/_cinder-service-cleaner.sh.tpl b/cinder/templates/bin/_cinder-service-cleaner.sh.tpl new file mode 100644 index 0000000000..cfd6d84b8b --- /dev/null +++ b/cinder/templates/bin/_cinder-service-cleaner.sh.tpl @@ -0,0 +1,31 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -xe + +# If any cinder service is down, then sleep for 2 times the report_interval +# to confirm service is still down. +DISABLED_SVC="$(cinder-manage service list | grep XXX | grep -v ':-)' || true)" +if [ ! -z "${DISABLED_SVC}" ]; then + sleep {{ .Values.jobs.service_cleaner.sleep_time }} +fi + +bins=($(cinder-manage service list | grep XXX | grep -v ":-)" | awk '{print $1 }')) +hosts=($(cinder-manage service list | grep XXX | grep -v ":-)" | awk '{print $2 }')) + +for i in "${!bins[@]}"; do + cinder-manage service remove "${bins[i]}" "${hosts[i]}" +done diff --git a/cinder/templates/configmap-bin.yaml b/cinder/templates/configmap-bin.yaml index 40700856be..855baadd9d 100644 --- a/cinder/templates/configmap-bin.yaml +++ b/cinder/templates/configmap-bin.yaml @@ -71,6 +71,8 @@ data: {{ tuple "bin/_storage-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} clean-secrets.sh: | {{ tuple "bin/_clean-secrets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + cinder-service-cleaner.sh: | +{{ tuple "bin/_cinder-service-cleaner.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- if .Values.bootstrap.enabled }} bootstrap.sh: | {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/cinder/templates/cron-job-service-cleaner.yaml b/cinder/templates/cron-job-service-cleaner.yaml new file mode 100644 index 0000000000..a884a1c087 --- /dev/null +++ b/cinder/templates/cron-job-service-cleaner.yaml @@ -0,0 +1,85 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_service_cleaner }} +{{- $envAll := . }} + +{{- $serviceAccountName := "cinder-service-cleaner" }} +{{ tuple $envAll "service_cleaner" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cinder-service-cleaner + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + schedule: {{ .Values.jobs.service_cleaner.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.service_cleaner.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.service_cleaner.history.failed }} + {{- if .Values.jobs.service_cleaner.starting_deadline }} + startingDeadlineSeconds: {{ .Values.jobs.service_cleaner.starting_deadline }} + {{- end }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "cinder" "service-cleaner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "cinder" "service-cleaner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "service_cleaner" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} +{{ if $envAll.Values.pod.tolerations.cinder.enabled }} +{{ tuple $envAll "cinder" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }} +{{ end }} + initContainers: +{{ tuple $envAll "service_cleaner" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: cinder-service-cleaner +{{ tuple $envAll "cinder_service_cleaner" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.service_cleaner | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "service_cleaner" "container" "cinder_service_cleaner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + env: + command: + - /tmp/cinder-service-cleaner.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: cinder-bin + mountPath: /tmp/cinder-service-cleaner.sh + subPath: cinder-service-cleaner.sh + readOnly: true + - name: cinder-etc + mountPath: /etc/cinder/cinder.conf + subPath: cinder.conf + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: cinder-etc + secret: + secretName: cinder-etc + defaultMode: 0444 + - name: cinder-bin + configMap: + name: cinder-bin + defaultMode: 0555 +{{- end }} diff --git a/cinder/values.yaml b/cinder/values.yaml index 18bcda55f5..b691b59809 100644 --- a/cinder/values.yaml +++ b/cinder/values.yaml @@ -55,6 +55,7 @@ images: cinder_scheduler: docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic cinder_volume: docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic cinder_volume_usage_audit: docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic + cinder_service_cleaner: docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic cinder_storage_init: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic cinder_backup: docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic cinder_backup_storage_init: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic @@ -68,6 +69,13 @@ images: - image_repo_sync jobs: + service_cleaner: + cron: "0 */1 * * *" + starting_deadline: 600 + history: + success: 3 + failed: 1 + sleep_time: 60 volume_usage_audit: cron: "5 * * * *" starting_deadline: 600 @@ -126,6 +134,13 @@ pod: cinder_scheduler: readOnlyRootFilesystem: true allowPrivilegeEscalation: false + service_cleaner: + pod: + runAsUser: 42424 + container: + cinder_service_cleaner: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false cinder_volume: pod: runAsUser: 42424 @@ -362,6 +377,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + service_cleaner: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" bootstrap: enabled: true @@ -893,7 +915,6 @@ conf: osapi_volume_workers: 1 glance_api_version: 2 os_region_name: RegionOne - host: cinder-volume-worker # NOTE(portdirect): the bind port should not be defined, and is manipulated # via the endpoints section. osapi_volume_listen_port: null @@ -1204,6 +1225,19 @@ dependencies: service: identity - endpoint: internal service: volumev3 + service_cleaner: + jobs: + - cinder-db-sync + - cinder-rabbit-init + services: + - endpoint: internal + service: oslo_messaging + - endpoint: internal + service: oslo_db + - endpoint: internal + service: identity + - endpoint: internal + service: volumev3 storage_init: jobs: null tests: @@ -1496,6 +1530,7 @@ manifests: configmap_bin: true configmap_etc: true cron_volume_usage_audit: true + cron_job_service_cleaner: true deployment_api: true deployment_backup: true deployment_scheduler: true diff --git a/cinder/values_overrides/train-ubuntu_bionic.yaml b/cinder/values_overrides/train-ubuntu_bionic.yaml index 71908adbb7..52139b2211 100644 --- a/cinder/values_overrides/train-ubuntu_bionic.yaml +++ b/cinder/values_overrides/train-ubuntu_bionic.yaml @@ -12,6 +12,7 @@ images: cinder_scheduler: "docker.io/openstackhelm/cinder:train-ubuntu_bionic" cinder_volume: "docker.io/openstackhelm/cinder:train-ubuntu_bionic" cinder_volume_usage_audit: "docker.io/openstackhelm/cinder:train-ubuntu_bionic" + cinder_service_cleaner: "docker.io/openstackhelm/cinder:train-ubuntu_bionic" cinder_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" cinder_backup: "docker.io/openstackhelm/cinder:train-ubuntu_bionic" cinder_backup_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" diff --git a/cinder/values_overrides/ussuri-ubuntu_bionic.yaml b/cinder/values_overrides/ussuri-ubuntu_bionic.yaml index ffa07b4eb6..d78ad8874c 100644 --- a/cinder/values_overrides/ussuri-ubuntu_bionic.yaml +++ b/cinder/values_overrides/ussuri-ubuntu_bionic.yaml @@ -12,6 +12,7 @@ images: cinder_scheduler: "docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic" cinder_volume: "docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic" cinder_volume_usage_audit: "docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic" + cinder_service_cleaner: "docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic" cinder_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" cinder_backup: "docker.io/openstackhelm/cinder:ussuri-ubuntu_bionic" cinder_backup_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" diff --git a/cinder/values_overrides/victoria-ubuntu_focal.yaml b/cinder/values_overrides/victoria-ubuntu_focal.yaml index 6cdf86389f..03b623f841 100644 --- a/cinder/values_overrides/victoria-ubuntu_focal.yaml +++ b/cinder/values_overrides/victoria-ubuntu_focal.yaml @@ -12,6 +12,7 @@ images: cinder_scheduler: "docker.io/openstackhelm/cinder:victoria-ubuntu_focal" cinder_volume: "docker.io/openstackhelm/cinder:victoria-ubuntu_focal" cinder_volume_usage_audit: "docker.io/openstackhelm/cinder:victoria-ubuntu_focal" + cinder_service_cleaner: "docker.io/openstackhelm/cinder:victoria-ubuntu_focal" cinder_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" cinder_backup: "docker.io/openstackhelm/cinder:victoria-ubuntu_focal" cinder_backup_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" diff --git a/cinder/values_overrides/wallaby-ubuntu_focal.yaml b/cinder/values_overrides/wallaby-ubuntu_focal.yaml index 9e5269d582..d5d8771a3f 100644 --- a/cinder/values_overrides/wallaby-ubuntu_focal.yaml +++ b/cinder/values_overrides/wallaby-ubuntu_focal.yaml @@ -12,6 +12,7 @@ images: cinder_scheduler: "docker.io/openstackhelm/cinder:wallaby-ubuntu_focal" cinder_volume: "docker.io/openstackhelm/cinder:wallaby-ubuntu_focal" cinder_volume_usage_audit: "docker.io/openstackhelm/cinder:wallaby-ubuntu_focal" + cinder_service_cleaner: "docker.io/openstackhelm/cinder:wallaby-ubuntu_focal" cinder_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" cinder_backup: "docker.io/openstackhelm/cinder:wallaby-ubuntu_focal" cinder_backup_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" diff --git a/cinder/values_overrides/xena-ubuntu_focal.yaml b/cinder/values_overrides/xena-ubuntu_focal.yaml index a05ce819fc..4c7b2abf8b 100644 --- a/cinder/values_overrides/xena-ubuntu_focal.yaml +++ b/cinder/values_overrides/xena-ubuntu_focal.yaml @@ -12,6 +12,7 @@ images: cinder_scheduler: "docker.io/openstackhelm/cinder:xena-ubuntu_focal" cinder_volume: "docker.io/openstackhelm/cinder:xena-ubuntu_focal" cinder_volume_usage_audit: "docker.io/openstackhelm/cinder:xena-ubuntu_focal" + cinder_service_cleaner: "docker.io/openstackhelm/cinder:xena-ubuntu_focal" cinder_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" cinder_backup: "docker.io/openstackhelm/cinder:xena-ubuntu_focal" cinder_backup_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" diff --git a/cinder/values_overrides/yoga-ubuntu_focal.yaml b/cinder/values_overrides/yoga-ubuntu_focal.yaml index b15a02e00b..165d28ad6b 100644 --- a/cinder/values_overrides/yoga-ubuntu_focal.yaml +++ b/cinder/values_overrides/yoga-ubuntu_focal.yaml @@ -12,6 +12,7 @@ images: cinder_scheduler: "docker.io/openstackhelm/cinder:yoga-ubuntu_focal" cinder_volume: "docker.io/openstackhelm/cinder:yoga-ubuntu_focal" cinder_volume_usage_audit: "docker.io/openstackhelm/cinder:yoga-ubuntu_focal" + cinder_service_cleaner: "docker.io/openstackhelm/cinder:yoga-ubuntu_focal" cinder_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" cinder_backup: "docker.io/openstackhelm/cinder:yoga-ubuntu_focal" cinder_backup_storage_init: "docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic" diff --git a/releasenotes/notes/cinder.yaml b/releasenotes/notes/cinder.yaml index 638dd0d4be..bccc581ebb 100644 --- a/releasenotes/notes/cinder.yaml +++ b/releasenotes/notes/cinder.yaml @@ -47,4 +47,5 @@ cinder: - 0.2.28 Use HTTP probe instead of TCP probe - 0.2.29 Add SYS_ADMIN capability in cinder-volume - 0.2.30 Specify a existing configmap name for external ceph configuration + - 0.2.31 Remove fixed node name from default values and add service cleaner cronjob ...