From 20feafa8d6de20a198057d36d569166118d96daa Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Fri, 17 Nov 2017 13:39:03 -0600 Subject: [PATCH] Update scheduler chart to OSH conventions Change-Id: I8ed49914a68c7ed03a199701d3e9a5f2dc10b7cf --- charts/scheduler/templates/bin/_anchor.tpl | 77 +++++++------------ charts/scheduler/templates/bin/_pre_stop.tpl | 15 ++++ charts/scheduler/templates/configmap-bin.yaml | 2 +- charts/scheduler/templates/configmap-etc.yaml | 32 ++------ charts/scheduler/templates/daemonset.yaml | 44 +++++++---- .../templates/etc/_kubeconfig.yaml.tpl | 36 +++++++++ .../etc/_kubernetes-scheduler.yaml.tpl | 49 ++++++++++++ charts/scheduler/templates/secret.yaml | 4 +- charts/scheduler/values.yaml | 66 ++++++++++++---- examples/basic/armada-resources.yaml | 20 ++--- examples/complete/armada-resources.yaml | 20 ++--- .../bootstrap-armada-config.yaml | 20 ++--- 12 files changed, 252 insertions(+), 133 deletions(-) create mode 100644 charts/scheduler/templates/etc/_kubeconfig.yaml.tpl create mode 100644 charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl diff --git a/charts/scheduler/templates/bin/_anchor.tpl b/charts/scheduler/templates/bin/_anchor.tpl index 1734fb2e..90f5def2 100644 --- a/charts/scheduler/templates/bin/_anchor.tpl +++ b/charts/scheduler/templates/bin/_anchor.tpl @@ -1,55 +1,35 @@ #!/bin/sh +{{/* +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} set -x -export MANIFEST_PATH=/host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml -export ETC_PATH=/host{{ .Values.scheduler.host_etc_path }} - -copy_etc_files() { - mkdir -p $ETC_PATH - cp /configmap/* /secret/* $ETC_PATH -} - -create_manifest() { - mkdir -p $(dirname $MANIFEST_PATH) - cat < $MANIFEST_PATH ---- -apiVersion: v1 -kind: Pod -metadata: - name: {{ .Values.service.name }} - namespace: {{ .Release.Namespace }} - labels: - {{ .Values.service.name }}-service: enabled -spec: - hostNetwork: true - containers: - - name: scheduler - image: {{ .Values.images.scheduler }} - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: - - {{ .Values.scheduler.command }} - - --leader-elect=true - - --kubeconfig=/etc/kubernetes/scheduler/kubeconfig.yaml - - --v=5 - - volumeMounts: - - name: etc - mountPath: /etc/kubernetes/scheduler - volumes: - - name: etc - hostPath: - path: {{ .Values.scheduler.host_etc_path }} -EODOC +compare_copy_files() { + {{- range .Values.anchor.files_to_copy }} + if [ ! -e /host{{ .dest }} ] || ! cmp -s {{ .source }} /host{{ .dest }}; then + mkdir -p $(dirname /host{{ .dest }}) + cp {{ .source }} /host{{ .dest }} + fi + {{- end }} } cleanup() { - rm -f $MANIFEST_PATH - rm -rf $ETC_PATH + {{- range .Values.anchor.files_to_copy }} + rm -f /host{{ .dest }} + {{- end }} } while true; do @@ -59,10 +39,9 @@ while true; do break fi - if [ ! -e $MANIFEST_PATH ]; then - copy_etc_files - create_manifest - fi + # Compare and replace files on Genesis host if needed + # Copy files to other master nodes + compare_copy_files sleep {{ .Values.anchor.period }} done diff --git a/charts/scheduler/templates/bin/_pre_stop.tpl b/charts/scheduler/templates/bin/_pre_stop.tpl index 37bb5c0f..769c1b9d 100644 --- a/charts/scheduler/templates/bin/_pre_stop.tpl +++ b/charts/scheduler/templates/bin/_pre_stop.tpl @@ -1,4 +1,19 @@ #!/bin/sh +{{/* +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} set -x diff --git a/charts/scheduler/templates/configmap-bin.yaml b/charts/scheduler/templates/configmap-bin.yaml index e2740643..5bde5676 100644 --- a/charts/scheduler/templates/configmap-bin.yaml +++ b/charts/scheduler/templates/configmap-bin.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Values.service.name }}-bin + name: kubernetes-scheduler-bin data: anchor: |+ {{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/scheduler/templates/configmap-etc.yaml b/charts/scheduler/templates/configmap-etc.yaml index 7a5ca093..22882d0a 100644 --- a/charts/scheduler/templates/configmap-etc.yaml +++ b/charts/scheduler/templates/configmap-etc.yaml @@ -2,29 +2,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Values.service.name }}-etc + name: kubernetes-scheduler-etc data: - kubeconfig.yaml: |- - --- - apiVersion: v1 - clusters: - - cluster: - server: https://{{ .Values.network.kubernetes_netloc }} - certificate-authority: cluster-ca.pem - name: kubernetes - contexts: - - context: - cluster: kubernetes - user: scheduler - name: scheduler@kubernetes - current-context: scheduler@kubernetes - kind: Config - preferences: {} - users: - - name: scheduler - user: - client-certificate: scheduler.pem - client-key: scheduler-key.pem - - cluster-ca.pem: {{ .Values.tls.ca | quote }} - scheduler.pem: {{ .Values.tls.cert | quote }} + cluster-ca.pem: {{ .Values.secrets.tls.ca | quote }} + kubeconfig.yaml: |+ +{{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + kubernetes-scheduler.yaml: |+ +{{ tuple "etc/_kubernetes-scheduler.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + scheduler.pem: {{ .Values.secrets.tls.cert | quote }} diff --git a/charts/scheduler/templates/daemonset.yaml b/charts/scheduler/templates/daemonset.yaml index 83048151..1c6ed399 100644 --- a/charts/scheduler/templates/daemonset.yaml +++ b/charts/scheduler/templates/daemonset.yaml @@ -1,18 +1,27 @@ +{{/* +Copyright 2017 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} --- apiVersion: "extensions/v1beta1" kind: DaemonSet metadata: - name: {{ .Values.service.name }}-anchor - labels: - application: kubernetes - component: kubernetes-scheduler-anchor + name: kubernetes-scheduler-anchor spec: - selector: - matchLabels: - {{ .Values.service.name | quote }}: anchor - updateStrategy: - rollingUpdate: - maxUnavailable: 1 +{{ tuple $envAll "scheduler" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} template: metadata: annotations: @@ -20,20 +29,23 @@ spec: configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} labels: - {{ .Values.service.name | quote }}: anchor +{{ tuple $envAll "kubernetes" "kubernetes-scheduler-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} spec: hostNetwork: true dnsPolicy: {{ .Values.anchor.dns_policy }} nodeSelector: - {{ .Values.anchor.node_selector.key }}: {{ .Values.anchor.node_selector.value }} + {{ .Values.labels.scheduler.node_selector_key }}: {{ .Values.labels.scheduler.node_selector_value }} tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule - key: CriticalAddonsOnly operator: Exists + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.scheduler.timeout }} containers: - name: anchor - image: {{ .Values.images.anchor }} + image: {{ .Values.images.tags.anchor }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.scheduler | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} command: - /tmp/bin/anchor lifecycle: @@ -54,16 +66,16 @@ spec: volumes: - name: bin configMap: - name: {{ .Values.service.name }}-bin + name: kubernetes-scheduler-bin defaultMode: 0555 - name: etc configMap: - name: {{ .Values.service.name }}-etc + name: kubernetes-scheduler-etc defaultMode: 0444 - name: host hostPath: path: / - name: secret secret: - secretName: {{ .Values.service.name }} + secretName: kubernetes-scheduler defaultMode: 0444 diff --git a/charts/scheduler/templates/etc/_kubeconfig.yaml.tpl b/charts/scheduler/templates/etc/_kubeconfig.yaml.tpl new file mode 100644 index 00000000..5281f6a1 --- /dev/null +++ b/charts/scheduler/templates/etc/_kubeconfig.yaml.tpl @@ -0,0 +1,36 @@ +{{/* +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +--- +apiVersion: v1 +clusters: +- cluster: + server: https://{{ .Values.network.kubernetes_netloc }} + certificate-authority: cluster-ca.pem + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: scheduler + name: scheduler@kubernetes +current-context: scheduler@kubernetes +kind: Config +preferences: {} +users: +- name: scheduler + user: + client-certificate: scheduler.pem + client-key: scheduler-key.pem diff --git a/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl b/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl new file mode 100644 index 00000000..7e6183eb --- /dev/null +++ b/charts/scheduler/templates/etc/_kubernetes-scheduler.yaml.tpl @@ -0,0 +1,49 @@ +{{/* +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +--- +apiVersion: v1 +kind: Pod +metadata: + name: kubernetes-scheduler + namespace: {{ .Release.Namespace }} + labels: + kubernetes-scheduler-service: enabled +spec: + hostNetwork: true + containers: + - name: scheduler + image: {{ .Values.images.tags.scheduler }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + {{- range .Values.command_prefix }} + - {{ . }} + {{- end }} + - --leader-elect=true + - --kubeconfig=/etc/kubernetes/scheduler/kubeconfig.yaml + + volumeMounts: + - name: etc + mountPath: /etc/kubernetes/scheduler + defaultMode: 0444 + volumes: + - name: etc + hostPath: + path: {{ .Values.scheduler.host_etc_path }} diff --git a/charts/scheduler/templates/secret.yaml b/charts/scheduler/templates/secret.yaml index 43a9546f..faafb03d 100644 --- a/charts/scheduler/templates/secret.yaml +++ b/charts/scheduler/templates/secret.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ .Values.service.name }} + name: kubernetes-scheduler type: Opaque data: - scheduler-key.pem: {{ .Values.tls.key | b64enc }} + scheduler-key.pem: {{ .Values.secrets.tls.key | b64enc }} diff --git a/charts/scheduler/values.yaml b/charts/scheduler/values.yaml index e3fdc90c..2d5e214d 100644 --- a/charts/scheduler/values.yaml +++ b/charts/scheduler/values.yaml @@ -1,28 +1,68 @@ +release_group: null + anchor: dns_policy: Default kubelet: manifest_path: /etc/kubernetes/manifests - node_selector: - key: kubernetes-scheduler - value: enabled period: 15 termination_grace_period: 3600 + files_to_copy: + - source: /configmap/cluster-ca.pem + dest: /etc/kubernetes/scheduler/cluster-ca.pem + - source: /configmap/scheduler.pem + dest: /etc/kubernetes/scheduler/scheduler.pem + - source: /configmap/kubeconfig.yaml + dest: /etc/kubernetes/scheduler/kubeconfig.yaml + - source: /secret/scheduler-key.pem + dest: /etc/kubernetes/scheduler/scheduler-key.pem + - source: /configmap/kubernetes-scheduler.yaml + dest: /etc/kubernetes/manifests/kubernetes-scheduler.yaml + +labels: + scheduler: + node_selector_key: kubernetes-scheduler + node_selector_value: enabled + +pod: + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + scheduler: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + termination_grace_period: + scheduler: + timeout: 3600 + resources: + enabled: false + scheduler: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" scheduler: - command: /scheduler host_etc_path: /etc/kubernetes/scheduler -service: - name: kubernetes-scheduler - -tls: - ca: placeholder - cert: placeholder - key: placeholder +secrets: + tls: + ca: placeholder + cert: placeholder + key: placeholder images: - anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 - scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + pull_policy: "IfNotPresent" network: kubernetes_netloc: 10.96.0.1 + +command_prefix: + - /scheduler + - --v=5 diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 625fdb29..818e87e5 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -722,21 +722,21 @@ metadata: name: kubernetes path: $ dest: - path: $.values.tls.ca + path: $.values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: scheduler path: $ dest: - path: $.values.tls.cert + path: $.values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: scheduler path: $ dest: - path: $.values.tls.key + path: $.values.secrets.tls.key data: chart_name: scheduler @@ -746,17 +746,19 @@ data: upgrade: no_hooks: true values: - tls: - ca: placeholder - cert: placeholder - key: placeholder + secrets: + tls: + ca: placeholder + cert: placeholder + key: placeholder network: kubernetes_netloc: apiserver.kubernetes.promenade:6443 images: - anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 - scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 source: type: local diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index 7c274453..65673a30 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -753,21 +753,21 @@ metadata: name: kubernetes path: $ dest: - path: $.values.tls.ca + path: $.values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: scheduler path: $ dest: - path: $.values.tls.cert + path: $.values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: scheduler path: $ dest: - path: $.values.tls.key + path: $.values.secrets.tls.key data: chart_name: scheduler @@ -777,17 +777,19 @@ data: upgrade: no_hooks: true values: - tls: - ca: placeholder - cert: placeholder - key: placeholder + secrets: + tls: + ca: placeholder + cert: placeholder + key: placeholder network: kubernetes_netloc: apiserver.kubernetes.promenade:6443 images: - anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 - scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + tags: + anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.8.0 source: type: local diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml index ad038957..716e8295 100644 --- a/tools/gate/config-templates/bootstrap-armada-config.yaml +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -722,21 +722,21 @@ metadata: name: kubernetes path: $ dest: - path: $.values.tls.ca + path: $.values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: scheduler path: $ dest: - path: $.values.tls.cert + path: $.values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: scheduler path: $ dest: - path: $.values.tls.key + path: $.values.secrets.tls.key data: chart_name: scheduler @@ -746,17 +746,19 @@ data: upgrade: no_hooks: true values: - tls: - ca: placeholder - cert: placeholder - key: placeholder + secrets: + tls: + ca: placeholder + cert: placeholder + key: placeholder network: kubernetes_netloc: apiserver.kubernetes.promenade:6443 images: - anchor: ${IMAGE_HYPERKUBE} - scheduler: ${IMAGE_HYPERKUBE} + tags: + anchor: ${IMAGE_HYPERKUBE} + scheduler: ${IMAGE_HYPERKUBE} source: type: local