diff --git a/charts/shipyard/templates/service-airflow-worker.yaml b/charts/shipyard/templates/service-airflow-worker.yaml new file mode 100644 index 00000000..65e68dbe --- /dev/null +++ b/charts/shipyard/templates/service-airflow-worker.yaml @@ -0,0 +1,30 @@ +{{/* +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_airflow_worker }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "airflow_worker" "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: worker + port: {{ tuple "airflow_worker" "internal" "airflow_worker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "airflow" "worker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/charts/shipyard/templates/service-discovery-airflow-worker.yaml b/charts/shipyard/templates/service-discovery-airflow-worker.yaml new file mode 100644 index 00000000..9cfa41e0 --- /dev/null +++ b/charts/shipyard/templates/service-discovery-airflow-worker.yaml @@ -0,0 +1,32 @@ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests.service_discovery_airflow_worker }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "airflow_worker" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + ports: + - name: worker + port: {{ tuple "airflow_worker" "internal" "airflow_worker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + clusterIP: None + selector: +{{ tuple $envAll "airflow" "worker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} + diff --git a/charts/shipyard/templates/deployment-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml similarity index 80% rename from charts/shipyard/templates/deployment-airflow-worker.yaml rename to charts/shipyard/templates/statefulset-airflow-worker.yaml index 4298481c..8d5abc0a 100644 --- a/charts/shipyard/templates/deployment-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- if .Values.manifests.deployment_airflow_worker }} +{{- if .Values.manifests.statefulset_airflow_worker }} {{- $envAll := . }} {{- $dependencies := .Values.dependencies.airflow_server }} -{{- $mounts_airflow_worker := .Values.pod.mounts.airflow_worker.airflow_worker }} -{{- $mounts_airflow_worker_init := .Values.pod.mounts.airflow_worker.init_container }} + {{- $serviceAccountName := "airflow-worker" }} {{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- @@ -52,27 +51,23 @@ roleRef: apiGroup: rbac.authorization.k8s.io --- apiVersion: apps/v1beta1 -kind: Deployment +kind: StatefulSet metadata: name: airflow-worker spec: + serviceName: {{ tuple "airflow_worker" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" replicas: {{ .Values.pod.replicas.airflow.worker }} -{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} template: metadata: labels: {{ tuple $envAll "airflow" "worker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: - configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }} spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - restartPolicy: Always - terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} initContainers: -{{ tuple $envAll $dependencies $mounts_airflow_worker_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{- if .Values.prod_environment }} - name: airflow-shipyard-init image: {{ .Values.images.tags.shipyard }} @@ -89,6 +84,20 @@ spec: mountPath: /tmp/airflow/dags/ - name: pod-shipyard-share-airflow-plugins mountPath: /tmp/airflow/plugins/ + - name: worker-perms + image: {{ .Values.images.tags.airflow }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 +{{ tuple $envAll $envAll.Values.pod.resources.airflow.worker | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - chown + - -R + - "airflow:airflow" + - {{ .Values.conf.airflow.core.base_log_folder }} + volumeMounts: + - name: airflow-logs + mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{- end }} containers: - name: airflow-worker @@ -102,7 +111,7 @@ spec: name: {{ .Values.secrets.postgresql_airflow_db.user }} key: AIRFLOW_DATABASE_URI ports: - - containerPort: {{ .Values.network.airflow.worker.port }} + - containerPort: {{ tuple "airflow_worker" "internal" "airflow_worker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} args: ["worker"] readinessProbe: tcpSocket: @@ -137,7 +146,6 @@ spec: {{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{ if $mounts_airflow_worker.volumeMounts }}{{ toYaml $mounts_airflow_worker.volumeMounts | indent 12 }}{{ end }} volumes: - name: airflow-etc configMap: @@ -156,8 +164,15 @@ spec: emptyDir: {} - name: pod-shipyard-share-airflow-plugins emptyDir: {} - - name: airflow-logs - emptyDir: {} + volumeClaimTemplates: + - metadata: + name: airflow-logs + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.volume_worker.size }} + storageClassName: {{ .Values.volume_worker.class_name }} {{ else }} - name: airflow-dags hostPath: @@ -175,5 +190,4 @@ spec: hostPath: path: /var/lib/docker {{- end }} -{{ if $mounts_airflow_worker.volumes }}{{ toYaml $mounts_airflow_worker.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 04a3f11d..c684521b 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -131,6 +131,10 @@ dependencies: - service: oslo_messaging endpoint: internal +volume_worker: + class_name: general + size: 5Gi + # typically overriden by environmental # values, but should include all endpoints # required by this chart @@ -211,6 +215,18 @@ endpoints: default: http host_fqdn_override: default: null + airflow_worker: + name: airflow-worker + hosts: + default: airflow-worker + discovery: airflow-worker-discovery + host_fqdn_override: + default: null + path: null + scheme: 'http' + port: + airflow_worker: + default: 8793 postgresql_shipyard_db: name: postgresql_shipyard_db auth: @@ -582,7 +598,7 @@ manifests: deployment_airflow_scheduler: true deployment_shipyard: true deployment_airflow_web: true - deployment_airflow_worker: true + statefulset_airflow_worker: true ingress_airflow_api: true ingress_shipyard_api: true job_shipyard_db_init: true @@ -600,3 +616,5 @@ manifests: service_shipyard: true service_shipyard_ingress: true service_airflow_web: true + service_airflow_worker: true + service_discovery_airflow_worker: true