From e20c067e50d18321868b0cb1156db761c04adc45 Mon Sep 17 00:00:00 2001 From: Robert Church Date: Tue, 5 Oct 2021 02:25:06 -0400 Subject: [PATCH] Add NoSchedule tolerations for application charts Add a toleration for the node-role.kubernetes.io/master:NoSchedule taint. This taint will be restored to all standard (non-AIO) master nodes to prevent user pods from being scheduled and run. These workloads will be scheduled and run on a worker node. This change will ensure that the rbd/cephfs provisioner and ceph audit pods will continue run on the master nodes (as designed). The following tests were executed: - Using an the existing app, remove the application, add the node-role.kubernetes.io/master:NoSchedule taint and confirmed that upon application-apply that the application will not apply as the pods will be in a Pending state. - With the existing application applied, add the node-role.kubernetes.io/master:NoSchedule taint and execute an application-update to a new application with these changes. Confirmed that the application updates successfully. Confirmed that toleration is present when describing the pod(s). Change-Id: I0a6368c717d336ac6c024bda596c283d2943285b Depends-On: https://review.opendev.org/c/starlingx/config/+/812629 Story: 2009232 Task: 43346 Signed-off-by: Robert Church --- .../k8sapp_platform/tests/test_plugins.py | 2 ++ .../templates/job-ceph-pools-audit.yaml | 4 ++-- .../templates/config-provisioner.yaml | 10 ++++++--- .../cephfs-provisioner/values.yaml | 21 ++++++++++++------ .../rbd-provisioner/templates/daemonset.yaml | 4 ++-- .../rbd-provisioner/templates/deployment.yaml | 6 ++--- .../templates/pre-install-check-ceph.yaml | 10 ++++++--- .../helm-charts/rbd-provisioner/values.yaml | 22 +++++++++++++------ .../stx-platform-helm/manifests/manifest.yaml | 17 ++++++++++++++ 9 files changed, 69 insertions(+), 27 deletions(-) diff --git a/python-k8sapp-platform/k8sapp_platform/k8sapp_platform/tests/test_plugins.py b/python-k8sapp-platform/k8sapp_platform/k8sapp_platform/tests/test_plugins.py index d96f670..a30c462 100644 --- a/python-k8sapp-platform/k8sapp_platform/k8sapp_platform/tests/test_plugins.py +++ b/python-k8sapp-platform/k8sapp_platform/k8sapp_platform/tests/test_plugins.py @@ -1,4 +1,6 @@ # +# Copyright (c) 2021 Wind River Systems, Inc. +# # SPDX-License-Identifier: Apache-2.0 # diff --git a/stx-platform-helm/stx-platform-helm/helm-charts/ceph-pools-audit/templates/job-ceph-pools-audit.yaml b/stx-platform-helm/stx-platform-helm/helm-charts/ceph-pools-audit/templates/job-ceph-pools-audit.yaml index 8fc35fc..85efe57 100644 --- a/stx-platform-helm/stx-platform-helm/helm-charts/ceph-pools-audit/templates/job-ceph-pools-audit.yaml +++ b/stx-platform-helm/stx-platform-helm/helm-charts/ceph-pools-audit/templates/job-ceph-pools-audit.yaml @@ -48,10 +48,10 @@ spec: restartPolicy: OnFailure nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} - {{- with .Values.tolerations }} +{{- with .Values.tolerations }} tolerations: {{ toYaml . | indent 12 }} - {{- end }} +{{- end }} volumes: - name: ceph-pools-bin configMap: diff --git a/stx-platform-helm/stx-platform-helm/helm-charts/cephfs-provisioner/templates/config-provisioner.yaml b/stx-platform-helm/stx-platform-helm/helm-charts/cephfs-provisioner/templates/config-provisioner.yaml index dd8880d..54755fc 100644 --- a/stx-platform-helm/stx-platform-helm/helm-charts/cephfs-provisioner/templates/config-provisioner.yaml +++ b/stx-platform-helm/stx-platform-helm/helm-charts/cephfs-provisioner/templates/config-provisioner.yaml @@ -92,7 +92,11 @@ spec: readOnly: true {{- end }} restartPolicy: OnFailure - {{- if .Values.global.nodeSelector }} +{{- if .Values.global.nodeSelector }} nodeSelector: - {{ .Values.global.nodeSelector | toYaml | trim | indent 8 }} - {{- end }} +{{ .Values.global.nodeSelector | toYaml | trim | indent 8 }} +{{- end }} +{{- with .Values.global.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end}} diff --git a/stx-platform-helm/stx-platform-helm/helm-charts/cephfs-provisioner/values.yaml b/stx-platform-helm/stx-platform-helm/helm-charts/cephfs-provisioner/values.yaml index ba985a5..69db6b5 100644 --- a/stx-platform-helm/stx-platform-helm/helm-charts/cephfs-provisioner/values.yaml +++ b/stx-platform-helm/stx-platform-helm/helm-charts/cephfs-provisioner/values.yaml @@ -25,13 +25,20 @@ global: # # Node tolerations for cephfs-provisioner scheduling to nodes with taints. # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - # Example: - # [ - # { - # "key": "node-role.kubernetes.io/master", - # "operator": "Exists" - # } - # ] + # Examples : + # tolerations: + # [ + # { + # key: "node-role.kubernetes.io/master", + # operator: "Exists", + # effect: "NoSchedule" + # } + # ] + # + # tolerations: + # - key: "node-role.kubernetes.io/master" + # operator: "Exists" + # effect: "NoSchedule" # tolerations: [] # If configured, resources will set the requests/limits field to the Pod. diff --git a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/daemonset.yaml b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/daemonset.yaml index 40d01ac..e5a0fe0 100644 --- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/daemonset.yaml +++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/daemonset.yaml @@ -26,9 +26,9 @@ spec: {{- if or .Values.global.rbac .Values.global.reuseRbac }} serviceAccountName: {{ .Values.rbac.serviceAccount }} {{- end }} -{{- if .Values.global.tolerations }} +{{- with .Values.global.tolerations }} tolerations: -{{ .Values.global.tolerations | toYaml | trim | indent 8 }} +{{ toYaml . | indent 8 }} {{- end }} {{- if .Values.global.nodeSelector }} nodeSelector: diff --git a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/deployment.yaml b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/deployment.yaml index 5bda622..31b3339 100644 --- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/deployment.yaml +++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/deployment.yaml @@ -47,10 +47,10 @@ spec: nodeSelector: {{ .Values.global.nodeSelector | toYaml | trim | indent 8 }} {{- end }} -{{- if .Values.global.tolerations }} +{{- with .Values.global.tolerations }} tolerations: -{{ .Values.global.tolerations | toYaml | trim | indent 8 }} -{{- end}} +{{ toYaml . | indent 8 }} +{{- end }} {{- if .Values.global.resources }} resources: {{ .Values.global.resources | toYaml | trim | indent 8 }} diff --git a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/pre-install-check-ceph.yaml b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/pre-install-check-ceph.yaml index f57bec2..26b70ef 100644 --- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/pre-install-check-ceph.yaml +++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/templates/pre-install-check-ceph.yaml @@ -192,10 +192,14 @@ spec: - name: config-volume-{{- $root.Values.global.name }} mountPath: {{ $mount }} {{- end }} - {{- if .Values.global.nodeSelector }} +{{- if $root.Values.global.nodeSelector }} nodeSelector: - {{ .Values.global.nodeSelector | toYaml | trim | indent 8 }} - {{- end }} +{{ $root.Values.global.nodeSelector | toYaml | trim | indent 8 }} +{{- end }} +{{- with $root.Values.global.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end}} --- # This ConfigMap is needed because we're not using ceph's helm chart diff --git a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/values.yaml b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/values.yaml index 04b7268..d90d9f7 100644 --- a/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/values.yaml +++ b/stx-platform-helm/stx-platform-helm/helm-charts/rbd-provisioner/values.yaml @@ -51,15 +51,23 @@ global: # # Node tolerations for rbd-volume-provisioner scheduling to nodes with taints. # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ - # Example: - # [ - # { - # "key": "node-role.kubernetes.io/master", - # "operator": "Exists" - # } - # ] + # Examples : + # tolerations: + # [ + # { + # key: "node-role.kubernetes.io/master", + # operator: "Exists", + # effect: "NoSchedule" + # } + # ] + # + # tolerations: + # - key: "node-role.kubernetes.io/master" + # operator: "Exists" + # effect: "NoSchedule" # tolerations: [] + # # If configured, resources will set the requests/limits field to the Pod. # Ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ # Example: diff --git a/stx-platform-helm/stx-platform-helm/manifests/manifest.yaml b/stx-platform-helm/stx-platform-helm/manifests/manifest.yaml index 576b41a..5b8a0a4 100644 --- a/stx-platform-helm/stx-platform-helm/manifests/manifest.yaml +++ b/stx-platform-helm/stx-platform-helm/manifests/manifest.yaml @@ -36,6 +36,12 @@ data: - type: job labels: app: rbd-provisioner + values: + global: + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" source: type: tar location: http://172.17.0.1:8080/helm_charts/stx-platform/rbd-provisioner-0.1.0.tgz @@ -65,6 +71,12 @@ data: - type: job labels: app: cephfs-provisioner + values: + global: + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" source: type: tar location: http://172.17.0.1:8080/helm_charts/stx-platform/cephfs-provisioner-0.1.0.tgz @@ -94,6 +106,11 @@ data: - type: job labels: app: ceph-pools-audit + values: + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" source: type: tar location: http://172.17.0.1:8080/helm_charts/stx-platform/ceph-pools-audit-0.1.0.tgz