From 9b9913d64a115d1a3484cd4623a0586257704fea Mon Sep 17 00:00:00 2001 From: Luna Das Date: Tue, 4 Feb 2020 19:55:00 +0530 Subject: [PATCH] Add Helm Charts for MetaController and DaemonJobController. These charts bootstraps a metacontroller on a Kubernetes cluster using the Helm package manager. This enables you to deploy custom controllers as service + deployment pairs. A DaemonJobController chart bootstraps the CompositeController and register DaemonJob CRD, the daemonjob controller executes DaemonJob(CR's) in kubernetes Cluster. Change-Id: Ic946f564ea1cf07e89c90a598e59230dc240950c --- daemonjob-controller/Chart.yaml | 16 + daemonjob-controller/requirements.yaml | 16 + .../templates/bin/_sync-hook.py.tpl | 105 +++++ .../templates/composite-controller.yaml | 33 ++ .../templates/configmap-bin.yaml | 25 ++ daemonjob-controller/templates/crd.yaml | 393 ++++++++++++++++++ .../templates/deployment.yaml | 60 +++ .../templates/job-image-repo-sync.yaml | 18 + daemonjob-controller/templates/service.yaml | 28 ++ daemonjob-controller/values.yaml | 109 +++++ .../values_overrides/apparmor.yaml | 5 + metacontroller/Chart.yaml | 24 ++ metacontroller/requirements.yaml | 16 + metacontroller/templates/crds.yaml | 125 ++++++ .../templates/job-image-repo-sync.yaml | 18 + metacontroller/templates/service.yaml | 32 ++ metacontroller/templates/statefulset.yaml | 96 +++++ metacontroller/values.yaml | 111 +++++ metacontroller/values_overrides/apparmor.yaml | 5 + .../deployment/common/daemonjob-controller.sh | 112 +++++ tools/deployment/common/metacontroller.sh | 56 +++ zuul.d/jobs.yaml | 19 + zuul.d/project.yaml | 2 + 23 files changed, 1424 insertions(+) create mode 100644 daemonjob-controller/Chart.yaml create mode 100644 daemonjob-controller/requirements.yaml create mode 100644 daemonjob-controller/templates/bin/_sync-hook.py.tpl create mode 100644 daemonjob-controller/templates/composite-controller.yaml create mode 100644 daemonjob-controller/templates/configmap-bin.yaml create mode 100644 daemonjob-controller/templates/crd.yaml create mode 100644 daemonjob-controller/templates/deployment.yaml create mode 100644 daemonjob-controller/templates/job-image-repo-sync.yaml create mode 100644 daemonjob-controller/templates/service.yaml create mode 100644 daemonjob-controller/values.yaml create mode 100644 daemonjob-controller/values_overrides/apparmor.yaml create mode 100644 metacontroller/Chart.yaml create mode 100644 metacontroller/requirements.yaml create mode 100644 metacontroller/templates/crds.yaml create mode 100644 metacontroller/templates/job-image-repo-sync.yaml create mode 100644 metacontroller/templates/service.yaml create mode 100644 metacontroller/templates/statefulset.yaml create mode 100644 metacontroller/values.yaml create mode 100644 metacontroller/values_overrides/apparmor.yaml create mode 100755 tools/deployment/common/daemonjob-controller.sh create mode 100755 tools/deployment/common/metacontroller.sh diff --git a/daemonjob-controller/Chart.yaml b/daemonjob-controller/Chart.yaml new file mode 100644 index 000000000..2186ea7bc --- /dev/null +++ b/daemonjob-controller/Chart.yaml @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: A Helm chart for DaemonjobController +name: daemonjob-controller +version: 0.1.0 diff --git a/daemonjob-controller/requirements.yaml b/daemonjob-controller/requirements.yaml new file mode 100644 index 000000000..5669e12cf --- /dev/null +++ b/daemonjob-controller/requirements.yaml @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/daemonjob-controller/templates/bin/_sync-hook.py.tpl b/daemonjob-controller/templates/bin/_sync-hook.py.tpl new file mode 100644 index 000000000..3c5b97d42 --- /dev/null +++ b/daemonjob-controller/templates/bin/_sync-hook.py.tpl @@ -0,0 +1,105 @@ +#!/usr/bin/env python +{{/* +Copyright 2019 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +import copy +from http.server import BaseHTTPRequestHandler, HTTPServer +import io +import json + + +def is_job_finished(job): + if 'status' in job: + desiredNumberScheduled = job['status'].get('desiredNumberScheduled', 1) + numberReady = job['status'].get('numberReady', 0) + if (desiredNumberScheduled == numberReady and + desiredNumberScheduled > 0): + return True + return False + + +def new_daemon(job): + daemon = copy.deepcopy(job) + daemon['apiVersion'] = 'apps/v1' + daemon['kind'] = 'DaemonSet' + daemon['metadata'] = {} + daemon['metadata']['name'] = '%s-dj' % (job['metadata']['name']) + daemon['metadata']['labels'] = copy.deepcopy( + job['spec']['template']['metadata']['labels']) + daemon['spec'] = {} + daemon['spec']['template'] = copy.deepcopy(job['spec']['template']) + daemon['spec']['template']['spec']['initContainers'] = copy.deepcopy( + job['spec']['template']['spec']['containers']) + daemon['spec']['template']['spec']['containers'] = [ + {'name': "pause", 'image': job['spec'].get( + 'pauseImage', 'gcr.io/google_containers/pause'), + 'resources': {'requests': {'cpu': '10m'}}}] + daemon['spec']['selector'] = {'matchLabels': copy.deepcopy( + job['spec']['template']['metadata']['labels'])} + + return daemon + + +class Controller(BaseHTTPRequestHandler): + def sync(self, job, children): + desired_status = {} + child = '%s-dj' % (job['metadata']['name']) + + # If the job already finished at some point, freeze the status, + # delete children, and take no further action. + if is_job_finished(job): + desired_status = copy.deepcopy(job['status']) + desired_status['conditions'] = [ + {'type': 'Complete', 'status': 'True'}] + return {'status': desired_status, 'children': []} + + # Compute status based on what we observed, + # before building desired state. + # Our .status is just a copy of the DaemonSet . + # status with extra fields. + desired_status = copy.deepcopy( + children['DaemonSet.apps/v1'].get(child, {}).get('status', {})) + if is_job_finished(children['DaemonSet.apps/v1'].get(child, {})): + desired_status['conditions'] = [ + {'type': 'Complete', 'status': 'True'}] + else: + desired_status['conditions'] = [ + {'type': 'Complete', 'status': 'False'}] + + # Always generate desired state for child if we reach this point. + # We should not delete children until after we know we've recorded + # completion in our status, which was the first check we did above. + desired_child = new_daemon(job) + return {'status': desired_status, 'children': [desired_child]} + + def do_POST(self): + observed = json.loads(self.rfile.read( + int(self.headers.get('Content-Length')))) + desired = self.sync(observed['parent'], observed['children']) + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.end_headers() + out = io.TextIOWrapper( + self.wfile, + encoding='utf-8', + line_buffering=False, + write_through=True, + ) + out.write(json.dumps(desired)) + out.detach() + + +HTTPServer(('', 80), Controller).serve_forever() diff --git a/daemonjob-controller/templates/composite-controller.yaml b/daemonjob-controller/templates/composite-controller.yaml new file mode 100644 index 000000000..b3a2523ca --- /dev/null +++ b/daemonjob-controller/templates/composite-controller.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ $groupName := .Values.crds.group_name }} +{{ $groupVersion := .Values.crds.group_version }} +{{ $groupVersionFormat := printf "%s/%s" $groupName $groupVersion }} +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: CompositeController +metadata: + name: daemonjob-controller +spec: + generateSelector: true + parentResource: + apiVersion: {{ $groupVersionFormat }} + resource: daemonjobs + childResources: + - apiVersion: apps/v1 + resource: daemonsets + hooks: + sync: + webhook: + url: http://daemonjob-controller.metacontroller/sync diff --git a/daemonjob-controller/templates/configmap-bin.yaml b/daemonjob-controller/templates/configmap-bin.yaml new file mode 100644 index 000000000..01fd461f8 --- /dev/null +++ b/daemonjob-controller/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: daemonjob-controller-bin + namespace: {{ .Release.Namespace }} +data: + sync.py: | +{{ tuple "bin/_sync-hook.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/daemonjob-controller/templates/crd.yaml b/daemonjob-controller/templates/crd.yaml new file mode 100644 index 000000000..0a8edf2e8 --- /dev/null +++ b/daemonjob-controller/templates/crd.yaml @@ -0,0 +1,393 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.crds_create }} +{{ $groupName := .Values.crds.group_name }} +{{ $groupVersion := .Values.crds.group_version }} +{{ $groupVersionFormat := printf "%s/%s" $groupName $groupVersion }} +{{ $crdName := printf "%s.%s" "daemonjobs" $groupName }} +{{- if not (.Capabilities.APIVersions.Has $groupVersionFormat) }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: {{ $crdName }} +spec: + group: {{ $groupName }} + versions: + - name: {{ $groupVersion }} + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + selector: + type: object + properties: + matchLabels: + type: object + additionalProperties: + type: string + template: + type: object + properties: + metadata: + type: object + properties: + annotations: + type: object + additionalProperties: + type: string + labels: + type: object + additionalProperties: + type: string + spec: + type: object + properties: + containers: + type: array + items: + type: object + properties: + name: + type: string + image: + type: string + imagePullPolicy: + type: string + command: + type: array + items: + type: string + workingDir: + type: string + lifecycle: + type: object + properties: + postStart: + type: object + properties: + exec: + type: object + properties: + command: + type: array + items: + type: string + httpGet: + type: object + properties: + host: + type: string + httpHeaders: + type: array + items: + type: object + properties: + name: + type: string + value: + type: string + path: + type: string + port: + type: string + scheme: + type: string + tcpSocket: + type: object + additionalProperties: + type: string + preStop: + type: object + properties: + exec: + type: object + properties: + command: + type: array + items: + type: string + httpGet: + type: object + properties: + host: + type: string + httpHeaders: + type: array + items: + type: object + properties: + name: + type: string + value: + type: string + path: + type: string + port: + type: string + scheme: + type: string + tcpSocket: + type: object + additionalProperties: + type: string + env: + type: array + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + additionalProperties: + type: string + fieldRef: + type: object + additionalProperties: + type: string + resourceFieldRef: + type: object + additionalProperties: + type: string + secretKeyRef: + type: object + additionalProperties: + type: string + envFrom: + type: array + items: + type: object + properties: + configMapKeyRef: + type: object + additionalProperties: + type: string + fieldRef: + type: object + additionalProperties: + type: string + resourceFieldRef: + type: object + additionalProperties: + type: string + secretKeyRef: + type: object + additionalProperties: + type: string + livenessProbe: + type: object + properties: + exec: + type: object + properties: + command: + type: array + items: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + startupProbe: + type: object + properties: + exec: + type: object + properties: + command: + type: array + items: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + securityContext: + type: object + properties: + allowPrivilegeEscalation: + type: boolean + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + type: integer + runAsNonRoot: + type: boolean + runAsUser: + type: integer + capabilities: + type: object + properties: + add: + type: array + items: + type: string + drop: + type: array + items: + type: string + seLinuxOptions: + type: object + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + windowsOptions: + type: object + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + ports: + type: array + items: + type: object + properties: + containerPort: + type: integer + hostIP: + type: string + hostPort: + type: integer + name: + type: string + protocol: + type: string + readinessProbe: + type: object + properties: + exec: + type: object + properties: + command: + type: array + items: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + resources: + type: object + properties: + requests: + type: object + properties: + cpu: + type: string + volumeMounts: + type: array + items: + type: object + properties: + mountPath: + type: string + name: + type: string + mountPropagation: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + volumes: + type: array + items: + type: object + properties: + name: + type: string + hostPath: + type: object + additionalProperties: + type: string + configMap: + type: object + additionalProperties: + type: string + restartPolicy: + type: string + tty: + type: boolean + terminationMessagePolicy: + type: string + terminationMessagePath: + type: string + stdinOnce: + type: boolean + stdin: + type: boolean + terminationGracePeriodSeconds: + type: integer + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + status: + type: string + type: + type: string + currentNumberScheduled: + type: integer + desiredNumberScheduled: + type: integer + numberMisscheduled: + type: integer + numberReady: + type: integer + numberUnavailable: + type: integer + observedGeneration: + type: integer + updatedNumberScheduled: + type: integer + subresources: + status: {} + scope: Namespaced + names: + plural: daemonjobs + singular: daemonjob + kind: DaemonJob + shortNames: ["dj"] +{{- end }} +{{- end }} diff --git a/daemonjob-controller/templates/deployment.yaml b/daemonjob-controller/templates/deployment.yaml new file mode 100644 index 000000000..4f7d856c5 --- /dev/null +++ b/daemonjob-controller/templates/deployment.yaml @@ -0,0 +1,60 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $serviceAccountName := "daemonjob-controller-serviceaccount" }} +{{ tuple $envAll "daemonjob_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: daemonjob-controller + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 4 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + namespace: {{ .Release.Namespace }} + labels: +{{ tuple $envAll "daemonjob-controller" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.daemonjob_controller }} + selector: + matchLabels: +{{ tuple $envAll "daemonjob-controller" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + annotations: +{{ dict "envAll" $envAll "podName" "daemonjob-controller" "containerNames" (list "controller") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + labels: +{{ tuple $envAll "daemonjob-controller" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + containers: + - name: controller +{{ tuple $envAll "python" | include "helm-toolkit.snippets.image" | indent 8 }} +{{ tuple $envAll $envAll.Values.pod.resources.daemonjob_controller | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }} + command: + - python + - /hooks/sync.py + volumeMounts: + - name: hooks + mountPath: /hooks + readOnly: true + volumes: + - name: hooks + configMap: + name: daemonjob-controller-bin + defaultMode: 0555 +{{- end }} diff --git a/daemonjob-controller/templates/job-image-repo-sync.yaml b/daemonjob-controller/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..b8a37270c --- /dev/null +++ b/daemonjob-controller/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "daemonjob-controller" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} \ No newline at end of file diff --git a/daemonjob-controller/templates/service.yaml b/daemonjob-controller/templates/service.yaml new file mode 100644 index 000000000..2e87db959 --- /dev/null +++ b/daemonjob-controller/templates/service.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "daemonjob_controller" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: 80 + selector: +{{ tuple $envAll "daemonjob-controller" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/daemonjob-controller/values.yaml b/daemonjob-controller/values.yaml new file mode 100644 index 000000000..b3c8a76fe --- /dev/null +++ b/daemonjob-controller/values.yaml @@ -0,0 +1,109 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for elasticsearch +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +images: + tags: + python: docker.io/python:3.6-slim + image_repo_sync: docker.io/docker:17.07.0 + pullPolicy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +crds: + group_name: ctl.example.com + group_version: v1 + +pod: + lifecycle: + upgrades: + deployments: + pod_replacement_strategy: RollingUpdate + revision_history: 3 + rolling_update: + max_surge: 3 + max_unavailable: 1 + resources: + enabled: false + daemonjob_controller: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + replicas: + daemonjob_controller: 1 + security_context: + daemonjob_controller: + pod: + runAsUser: 34356 + runAsNonRoot: true + container: + controller: + runAsUser: 34356 + readOnlyRootFilesystem: true + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + daemonjob_controller: + hosts: + default: daemonjob-controller + host_fqdn_override: + default: null + port: + http: + default: 80 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - daemonjob-controller-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + daemonjob_controller: + services: null + +manifests: + deployment: true + crds_create: true + job_image_repo_sync: true + configmap_bin: true + service: true diff --git a/daemonjob-controller/values_overrides/apparmor.yaml b/daemonjob-controller/values_overrides/apparmor.yaml new file mode 100644 index 000000000..39922e5fb --- /dev/null +++ b/daemonjob-controller/values_overrides/apparmor.yaml @@ -0,0 +1,5 @@ +pod: + mandatory_access_control: + type: apparmor + daemonjob-controller: + controller: localhost/docker-default \ No newline at end of file diff --git a/metacontroller/Chart.yaml b/metacontroller/Chart.yaml new file mode 100644 index 000000000..d2404c0ac --- /dev/null +++ b/metacontroller/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: A Helm chart for Metacontroller +name: metacontroller +version: 0.1.0 +home: https://metacontroller.app/ +keywords: + - CRDs + - metacontroller +sources: + - https://github.com/GoogleCloudPlatform/metacontroller +maintainers: + - name: OpenStack-Helm Authors diff --git a/metacontroller/requirements.yaml b/metacontroller/requirements.yaml new file mode 100644 index 000000000..5669e12cf --- /dev/null +++ b/metacontroller/requirements.yaml @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/metacontroller/templates/crds.yaml b/metacontroller/templates/crds.yaml new file mode 100644 index 000000000..c98506e71 --- /dev/null +++ b/metacontroller/templates/crds.yaml @@ -0,0 +1,125 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.crds }} +{{- if not (.Capabilities.APIVersions.Has "metacontroller.k8s.io/v1alpha1") }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: compositecontrollers.metacontroller.k8s.io + annotations: + "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/78458" +spec: + group: metacontroller.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + generateSelector: + type: boolean + parentResource: + type: object + properties: + apiVersion: + type: string + resource: + type: string + childResources: + type: array + items: + type: object + properties: + apiVersion: + type: string + resource: + type: string + hooks: + type: object + properties: + sync: + type: object + properties: + webhook: + type: object + properties: + url: + type: string + scope: Cluster + names: + plural: compositecontrollers + singular: compositecontroller + kind: CompositeController + shortNames: + - cc + - cctl +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: decoratorcontrollers.metacontroller.k8s.io + annotations: + "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/78458" +spec: + group: metacontroller.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + scope: Cluster + names: + plural: decoratorcontrollers + singular: decoratorcontroller + kind: DecoratorController + shortNames: + - dec + - decorators +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: controllerrevisions.metacontroller.k8s.io + annotations: + "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/78458" +spec: + group: metacontroller.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + scope: Namespaced + names: + plural: controllerrevisions + singular: controllerrevision + kind: ControllerRevision +{{- end }} +{{- end }} diff --git a/metacontroller/templates/job-image-repo-sync.yaml b/metacontroller/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..7cc55d2f6 --- /dev/null +++ b/metacontroller/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "metacontroller" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} \ No newline at end of file diff --git a/metacontroller/templates/service.yaml b/metacontroller/templates/service.yaml new file mode 100644 index 000000000..62674a661 --- /dev/null +++ b/metacontroller/templates/service.yaml @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "metacontroller" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + namespace: {{ .Release.Namespace }} + labels: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + clusterIP: None + ports: + - name: metacontroller + port: {{ tuple "metacontroller" "internal" "metacontroller" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/metacontroller/templates/statefulset.yaml b/metacontroller/templates/statefulset.yaml new file mode 100644 index 000000000..81da00032 --- /dev/null +++ b/metacontroller/templates/statefulset.yaml @@ -0,0 +1,96 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $serviceAccountName := "metacontroller-serviceaccount" }} +{{ tuple $envAll "metacontroller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +{{ $controllerName := printf "%s-%s" .Release.Namespace $serviceAccountName }} +--- +{{- if .Values.manifests.rbac }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $controllerName }} +rules: +- apiGroups: + - "*" + resources: + - "*" + verbs: + - "*" +{{- end }} +--- +{{- if .Values.manifests.rbac }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $controllerName }} +subjects: +- kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $controllerName }} + apiGroup: rbac.authorization.k8s.io +{{- end }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: metacontroller + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 4 }} + namespace: {{ .Release.Namespace }} + labels: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + serviceName: {{ tuple "metacontroller" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" + affinity: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 4 }} + replicas: {{ .Values.pod.replicas.metacontroller }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.server.timeout | default "30" }} + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value | quote }} + template: + metadata: + labels: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "metacontroller" "containerNames" (list "metacontroller") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" . "application" "metacontroller" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + containers: + - name: metacontroller +{{ tuple $envAll "metacontroller" | include "helm-toolkit.snippets.image" | indent 8 }} +{{ tuple $envAll $envAll.Values.pod.resources.metacontroller | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }} +{{ dict "envAll" $envAll "application" "metacontroller" "container" "metacontroller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }} + ports: + - name: metacontroller + containerPort: {{ tuple "metacontroller" "internal" "metacontroller" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + command: + - /usr/bin/metacontroller + args: + - --logtostderr + - -v=6 + - --discovery-interval=20s +{{- end }} diff --git a/metacontroller/values.yaml b/metacontroller/values.yaml new file mode 100644 index 000000000..63ce43c20 --- /dev/null +++ b/metacontroller/values.yaml @@ -0,0 +1,111 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for elasticsearch +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +images: + tags: + metacontroller: docker.io/metacontroller/metacontroller:v0.4.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - metacontroller-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry +pod: + lifecycle: + termination_grace_period: + server: + timeout: 600 + resources: + enabled: false + metacontroller: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + replicas: + metacontroller: 1 + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + security_context: + metacontroller: + container: + metacontroller: + runAsUser: 34356 + readOnlyRootFilesystem: true + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + metacontroller: + hosts: + default: metacontroller + host_fqdn_override: + default: null + port: + metacontroller: + default: 8083 + +manifests: + service: true + statefulset: true + job_image_repo_sync: true + crds: true + rbac: true + + diff --git a/metacontroller/values_overrides/apparmor.yaml b/metacontroller/values_overrides/apparmor.yaml new file mode 100644 index 000000000..f194450b0 --- /dev/null +++ b/metacontroller/values_overrides/apparmor.yaml @@ -0,0 +1,5 @@ +pod: + mandatory_access_control: + type: apparmor + metacontroller: + metacontroller: localhost/docker-default \ No newline at end of file diff --git a/tools/deployment/common/daemonjob-controller.sh b/tools/deployment/common/daemonjob-controller.sh new file mode 100755 index 000000000..fa9d198e7 --- /dev/null +++ b/tools/deployment/common/daemonjob-controller.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -xe + +namespace="metacontroller" +: ${HELM_ARGS_DAEMONJOB_CONTROLLER:="$(./tools/deployment/common/get-values-overrides.sh daemonjob-controller)"} + +#NOTE: Lint and package chart +make daemonjob-controller + +#NOTE: Deploy command +helm upgrade --install daemonjob-controller ./daemonjob-controller \ + --namespace=$namespace \ + ${HELM_ARGS_DAEMONJOB_CONTROLLER} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh daemonjob-controller + +#NOTE: CompositeController succesfully deployed +composite_controller_cr=$(kubectl get compositecontrollers | awk '{print $1}') +echo "$composite_controller_cr, a CompositeController created succesfully" + +#NOTE: Check crd of APIGroup ctl.example.com +daemonjob_crd=$(kubectl get crd | awk '/ctl.example.com/{print $1}') +echo "$daemonjob_crd is succesfully created" + +#NOTE: Check daemonjob_controller is running +pod=$(kubectl get pods -n $namespace | awk '/daemonjob-controller/{print $1}') +daemonjob_controller_status=$(kubectl get pods -n $namespace | awk '/daemonjob-controller/{print $3}') + +NEXT_WAIT_TIME=0 +until [[ $daemonjob_controller_status == 'Running' ]] || [ $NEXT_WAIT_TIME -eq 5 ]; do + daemonjob_controller_status=$(kubectl get pods -n $namespace | awk '/daemonjob-controller/{print $3}') + echo "DaemonjobController is not still up and running" + sleep 20 + NEXT_WAIT_TIME=$((NEXT_WAIT_TIME+1)) +done + +#NOTE: Validate DaemonjobController Deployment info +helm status daemonjob-controller + +#NOTE: Create sample-daemonjob.yaml +tee /tmp/sample-daemonjob.yaml << EOF +apiVersion: ctl.example.com/v1 +kind: DaemonJob +metadata: + name: hello-world +spec: + template: + metadata: + labels: + app: hello-world + annotations: + container.apparmor.security.beta.kubernetes.io/hello-world: localhost/docker-default + spec: + containers: + - name: hello-world + image: busybox + command: ["sh", "-c", "echo 'Hello world' && sleep 120"] + resources: + requests: + cpu: 10m + terminationGracePeriodSeconds: 10 +EOF + +dj="daemonjobs" + +#NOTE: Deploy daemonjob +kubectl apply -f /tmp/sample-daemonjob.yaml + +#NOTE: Wait for successful completion +NEXT_WAIT_TIME=0 +echo "Wait for successful completion..." +until [[ "$(kubectl get $dj hello-world -o 'jsonpath={.status.conditions[0].status}')" == "True" ]] || [ $NEXT_WAIT_TIME -eq 5 ]; do + daemonset_pod=$(kubectl get pods | awk '/hello-world-dj/{print $1}') + if [ -z "$daemonset_pod" ] + then + echo "Child resource daemonset not yet created" + else + daemonset_pod_status=$(kubectl get pods | awk '/hello-world-dj/{print $3}') + if [ $daemonset_pod_status == 'Init:0/1' ]; then + init_container_status=$(kubectl get pod $daemonset_pod -o 'jsonpath={.status.initContainerStatuses[0].state.running}') + if [ ! -z "$init_container_status" ]; then + expected_log=$(kubectl logs $daemonset_pod -c hello-world) + if [ $expected_log == 'Hello world' ]; then + echo "Strings are equal." && break + fi + fi + fi + fi + sleep 20 + NEXT_WAIT_TIME=$((NEXT_WAIT_TIME+1)) +done + +#NOTE: Check that DaemonSet gets cleaned up after finishing +NEXT_WAIT_TIME=0 +echo "Check that DaemonSet gets cleaned up after finishing..." +until [[ "$(kubectl get daemonset hello-world-dj 2>&1)" =~ NotFound ]] || [ $NEXT_WAIT_TIME -eq 5 ]; do + sleep 20 + NEXT_WAIT_TIME=$((NEXT_WAIT_TIME+1)) +done diff --git a/tools/deployment/common/metacontroller.sh b/tools/deployment/common/metacontroller.sh new file mode 100755 index 000000000..c0ad04491 --- /dev/null +++ b/tools/deployment/common/metacontroller.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -xe + +namespace="metacontroller" +: ${HELM_ARGS_METACONTROLLER:="$(./tools/deployment/common/get-values-overrides.sh metacontroller)"} + +#NOTE: Lint and package chart +make metacontroller + +#NOTE: Check no crd exists of APIGroup metacontroller.k8s.io +crds=$(kubectl get crd | awk '/metacontroller.k8s.io/{print $1}') + +if [ -z "$crds" ]; then + echo "No crd exists of APIGroup metacontroller.k8s.io" +fi + +#NOTE: Deploy command +helm upgrade --install metacontroller ./metacontroller \ + --namespace=$namespace \ + --set pod.replicas.metacontroller=3 \ + ${HELM_ARGS_METACONTROLLER} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh metacontroller + +#NOTE: Check crds of APIGroup metacontroller.k8s.io successfully created +crds=$(kubectl get crd | awk '/metacontroller.k8s.io/{print $1}') + +COUNTER=0 +for i in $crds +do + case $i in + "compositecontrollers.metacontroller.k8s.io") COUNTER=$((COUNTER+1));; + "controllerrevisions.metacontroller.k8s.io") COUNTER=$((COUNTER+1));; + "decoratorcontrollers.metacontroller.k8s.io") COUNTER=$((COUNTER+1));; + *) echo "This is a wrong crd!!!";; + esac +done + +if test $COUNTER -eq 3; then + echo "crds created succesfully" +fi + +helm status metacontroller diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 81775b483..401542944 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -291,6 +291,25 @@ - ./tools/deployment/apparmor/120-openvswitch.sh - ./tools/deployment/apparmor/130-postgresql.sh +- job: + name: openstack-helm-infra-metacontroller + parent: openstack-helm-infra-functional + timeout: 7200 + pre-run: playbooks/osh-infra-upgrade-host.yaml + run: playbooks/osh-infra-gate-runner.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + nodeset: openstack-helm-single-node + vars: + osh_params: + container_distro_name: ubuntu + container_distro_version: bionic + feature_gates: apparmor + gate_scripts: + - ./tools/deployment/common/000-install-packages.sh + - ./tools/deployment/common/005-deploy-k8s.sh + - ./tools/deployment/common/metacontroller.sh + - ./tools/deployment/common/daemonjob-controller.sh + - job: name: openstack-helm-infra-openstack-support parent: openstack-helm-infra-functional diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 571842824..e55233375 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -43,6 +43,8 @@ voting: false - openstack-helm-infra-local-storage: voting: false + - openstack-helm-infra-metacontroller: + voting: false gate: jobs: - openstack-helm-lint