From 9725b0f337ebdc7523973b50afecbf4caf2b7e6c Mon Sep 17 00:00:00 2001 From: Bryan Strassner Date: Thu, 17 Jan 2019 13:55:40 -0600 Subject: [PATCH] Build workflows into Airflow image Changes to make the docker image build to include the workflows from Shipyard, rather than adding them to the container during Helm install of Shipyard. This also removes the "prod" switch, as it is now always built the same way, with the workflows in place. Change-Id: I4acd6195cbec32193e15621e75ccaeb9879455f5 --- .../bin/_airflow-shipyard-init.sh.tpl | 23 ------- .../templates/configmap-airflow-bin.yaml | 2 - .../deployment-airflow-scheduler.yaml | 44 ------------ .../templates/deployment-airflow-web.yaml | 44 ------------ .../templates/statefulset-airflow-worker.yaml | 69 ------------------- charts/shipyard/values.yaml | 6 -- images/airflow/Dockerfile | 16 ++++- 7 files changed, 15 insertions(+), 189 deletions(-) delete mode 100644 charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl diff --git a/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl b/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl deleted file mode 100644 index ef72f5c8..00000000 --- a/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -set -ex - -cp -vaR /home/shipyard/shipyard/shipyard_airflow/dags/*.py /tmp/airflow/dags/ -cp -vaR /home/shipyard/shipyard/shipyard_airflow/plugins/*.py /tmp/airflow/plugins/ diff --git a/charts/shipyard/templates/configmap-airflow-bin.yaml b/charts/shipyard/templates/configmap-airflow-bin.yaml index 12ab0f1f..18500224 100644 --- a/charts/shipyard/templates/configmap-airflow-bin.yaml +++ b/charts/shipyard/templates/configmap-airflow-bin.yaml @@ -27,6 +27,4 @@ data: {{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} airflow-db-sync.sh: |+ {{ tuple "bin/_airflow-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - airflow-shipyard-init.sh: |+ -{{ tuple "bin/_airflow-shipyard-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/charts/shipyard/templates/deployment-airflow-scheduler.yaml b/charts/shipyard/templates/deployment-airflow-scheduler.yaml index 8c6c597f..ff278aa2 100644 --- a/charts/shipyard/templates/deployment-airflow-scheduler.yaml +++ b/charts/shipyard/templates/deployment-airflow-scheduler.yaml @@ -53,23 +53,6 @@ spec: terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} initContainers: {{ tuple $envAll "airflow_server" $mounts_airflow_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} -{{- if .Values.prod_environment }} - - name: airflow-shipyard-init - image: {{ .Values.images.tags.shipyard }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/airflow-shipyard-init.sh - volumeMounts: - - name: airflow-bin - mountPath: /tmp/airflow-shipyard-init.sh - subPath: airflow-shipyard-init.sh - readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: /tmp/airflow/dags/ - - name: pod-shipyard-share-airflow-plugins - mountPath: /tmp/airflow/plugins/ -{{- end }} containers: - name: airflow-scheduler image: {{ .Values.images.tags.airflow }} @@ -88,21 +71,10 @@ spec: mountPath: {{ .Values.conf.airflow_config_file.path }} subPath: airflow.cfg readOnly: true -{{- if .Values.prod_environment }} - name: shipyard-etc mountPath: /usr/local/airflow/plugins/shipyard.conf subPath: shipyard.conf readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: pod-shipyard-share-airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{ else }} - - name: airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{ if $mounts_airflow_scheduler.volumeMounts }}{{ toYaml $mounts_airflow_scheduler.volumeMounts | indent 12 }}{{ end }} @@ -111,7 +83,6 @@ spec: configMap: name: airflow-etc defaultMode: 0444 -{{- if .Values.prod_environment }} - name: shipyard-etc configMap: name: shipyard-etc @@ -120,22 +91,7 @@ spec: configMap: name: airflow-bin defaultMode: 0555 - - name: pod-shipyard-share-airflow-dags - emptyDir: {} - - name: pod-shipyard-share-airflow-plugins - emptyDir: {} - name: airflow-logs emptyDir: {} -{{ else }} - - name: airflow-dags - hostPath: - path: {{ .Values.pod.mounts.dag_path }} - - name: airflow-plugins - hostPath: - path: {{ .Values.pod.mounts.plugin_path }} - - name: airflow-logs - hostPath: - path: {{ .Values.pod.mounts.log_path }} -{{- end }} {{ if $mounts_airflow_scheduler.volumes }}{{ toYaml $mounts_airflow_scheduler.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/charts/shipyard/templates/deployment-airflow-web.yaml b/charts/shipyard/templates/deployment-airflow-web.yaml index faaecb62..648f5564 100644 --- a/charts/shipyard/templates/deployment-airflow-web.yaml +++ b/charts/shipyard/templates/deployment-airflow-web.yaml @@ -44,23 +44,6 @@ spec: terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} initContainers: {{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} -{{- if .Values.prod_environment }} - - name: airflow-shipyard-init - image: {{ .Values.images.tags.shipyard }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/airflow-shipyard-init.sh - volumeMounts: - - name: airflow-bin - mountPath: /tmp/airflow-shipyard-init.sh - subPath: airflow-shipyard-init.sh - readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: /tmp/airflow/dags/ - - name: pod-shipyard-share-airflow-plugins - mountPath: /tmp/airflow/plugins/ -{{- end }} containers: - name: airflow-web image: {{ .Values.images.tags.airflow }} @@ -83,21 +66,10 @@ spec: mountPath: {{ .Values.conf.airflow_config_file.path }} subPath: airflow.cfg readOnly: true -{{- if .Values.prod_environment }} - name: shipyard-etc mountPath: /usr/local/airflow/plugins/shipyard.conf subPath: shipyard.conf readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: pod-shipyard-share-airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{ else }} - - name: airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }} @@ -106,7 +78,6 @@ spec: configMap: name: airflow-etc defaultMode: 0444 -{{- if .Values.prod_environment }} - name: shipyard-etc configMap: name: shipyard-etc @@ -115,22 +86,7 @@ spec: configMap: name: airflow-bin defaultMode: 0555 - - name: pod-shipyard-share-airflow-dags - emptyDir: {} - - name: pod-shipyard-share-airflow-plugins - emptyDir: {} - name: airflow-logs emptyDir: {} -{{ else }} - - name: airflow-dags - hostPath: - path: {{ .Values.pod.mounts.dag_path }} - - name: airflow-plugins - hostPath: - path: {{ .Values.pod.mounts.plugin_path }} - - name: airflow-logs - hostPath: - path: {{ .Values.pod.mounts.log_path }} -{{- end }} {{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml index a53473d7..c71b63eb 100644 --- a/charts/shipyard/templates/statefulset-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -79,22 +79,6 @@ spec: {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }} initContainers: {{ tuple $envAll "airflow_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} -{{- if .Values.prod_environment }} - - name: airflow-shipyard-init - image: {{ .Values.images.tags.shipyard }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/airflow-shipyard-init.sh - volumeMounts: - - name: airflow-bin - mountPath: /tmp/airflow-shipyard-init.sh - subPath: airflow-shipyard-init.sh - readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: /tmp/airflow/dags/ - - name: pod-shipyard-share-airflow-plugins - mountPath: /tmp/airflow/plugins/ - name: worker-perms image: {{ .Values.images.tags.airflow }} imagePullPolicy: {{ .Values.images.pull_policy }} @@ -109,7 +93,6 @@ spec: volumeMounts: - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{- end }} containers: - name: airflow-scheduler image: {{ .Values.images.tags.airflow }} @@ -128,21 +111,10 @@ spec: mountPath: {{ .Values.conf.airflow_config_file.path }} subPath: airflow.cfg readOnly: true -{{- if .Values.prod_environment }} - name: shipyard-etc mountPath: /usr/local/airflow/plugins/shipyard.conf subPath: shipyard.conf readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: pod-shipyard-share-airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{ else }} - - name: airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} - name: airflow-worker @@ -166,30 +138,12 @@ spec: mountPath: {{ .Values.conf.airflow_config_file.path }} subPath: airflow.cfg readOnly: true -{{- if .Values.prod_environment }} - name: shipyard-etc mountPath: /usr/local/airflow/plugins/shipyard.conf subPath: shipyard.conf readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: pod-shipyard-share-airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{ else }} - - name: airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} - - name: docker - mountPath: /var/run - readOnly: false - - name: pod-var-lib-docker - mountPath: /var/lib/docker - readOnly: false -{{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{- if .Values.prod_environment }} - name: airflow-logrotate image: {{ .Values.images.tags.airflow }} imagePullPolicy: {{ .Values.images.pull_policy }} @@ -214,13 +168,11 @@ spec: volumeMounts: - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{- end }} volumes: - name: airflow-etc configMap: name: airflow-etc defaultMode: 0444 -{{- if .Values.prod_environment }} - name: shipyard-etc configMap: name: shipyard-etc @@ -229,10 +181,6 @@ spec: configMap: name: airflow-bin defaultMode: 0555 - - name: pod-shipyard-share-airflow-dags - emptyDir: {} - - name: pod-shipyard-share-airflow-plugins - emptyDir: {} volumeClaimTemplates: - metadata: name: airflow-logs @@ -242,21 +190,4 @@ spec: requests: storage: {{ .Values.volume_worker.size }} storageClassName: {{ .Values.volume_worker.class_name }} -{{ else }} - - name: airflow-dags - hostPath: - path: {{ .Values.pod.mounts.dag_path }} - - name: airflow-plugins - hostPath: - path: {{ .Values.pod.mounts.plugin_path }} - - name: airflow-logs - hostPath: - path: {{ .Values.pod.mounts.log_path }} - - name: docker - hostPath: - path: /var/run - - name: pod-var-lib-docker - hostPath: - path: /var/lib/docker -{{- end }} {{- end }} diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 969ad4b3..998a4295 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -15,9 +15,6 @@ # This file provides defaults for shipyard and airflow -# Indicate whether it is production or development environment -prod_environment: true - labels: job: node_selector_key: ucp-control-plane @@ -701,9 +698,6 @@ conf: # End of Airflow config options pod: mounts: - dag_path: /home/ubuntu/workbench/dags - plugin_path: /home/ubuntu/workbench/plugins - log_path: /home/ubuntu/workbench/logs airflow_scheduler: # TODO: This is only used if the standalone scheduler is enabled. airflow_scheduler: diff --git a/images/airflow/Dockerfile b/images/airflow/Dockerfile index 8652f158..6b130962 100644 --- a/images/airflow/Dockerfile +++ b/images/airflow/Dockerfile @@ -104,7 +104,7 @@ ENV PBR_VERSION 0.1a1 # Shipyard # -# Shipyard provides core functionality used by the airflow plugins/operators +# Shipyard provides core functionality used by the Airflow plugins/operators # Since Shipyard and Airflow are built together as images, this should prevent # stale or out-of-date code between these parts. # Shipyard requirements, source and installation @@ -115,6 +115,20 @@ COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/ RUN cd /tmp/shipyard \ && python3 setup.py install +# Note: The value for the dags and plugins directories that are sourced +# from the values.yaml of the Shipyard Helm chart need to align with these +# directories. If they do not, airflow will not find the intended dags and +# plugins. +# +# Note: In the case of building images using the provided Makefile, a test is +# run against the built-in dags provided with Airflow. Since there is no Helm +# chart to reconfigure the airflow.cfg with these directories, these dags and +# plugins are not known to Airflow during the image test. +# +# Copy the plugins and dags that will be used by this Airflow image: +COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/ +COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/ + # Set work directory USER airflow WORKDIR ${AIRFLOW_HOME}