diff --git a/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl b/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl deleted file mode 100644 index ef72f5c8..00000000 --- a/charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -set -ex - -cp -vaR /home/shipyard/shipyard/shipyard_airflow/dags/*.py /tmp/airflow/dags/ -cp -vaR /home/shipyard/shipyard/shipyard_airflow/plugins/*.py /tmp/airflow/plugins/ diff --git a/charts/shipyard/templates/configmap-airflow-bin.yaml b/charts/shipyard/templates/configmap-airflow-bin.yaml index 12ab0f1f..18500224 100644 --- a/charts/shipyard/templates/configmap-airflow-bin.yaml +++ b/charts/shipyard/templates/configmap-airflow-bin.yaml @@ -27,6 +27,4 @@ data: {{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} airflow-db-sync.sh: |+ {{ tuple "bin/_airflow-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - airflow-shipyard-init.sh: |+ -{{ tuple "bin/_airflow-shipyard-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/charts/shipyard/templates/deployment-airflow-scheduler.yaml b/charts/shipyard/templates/deployment-airflow-scheduler.yaml index 8c6c597f..ff278aa2 100644 --- a/charts/shipyard/templates/deployment-airflow-scheduler.yaml +++ b/charts/shipyard/templates/deployment-airflow-scheduler.yaml @@ -53,23 +53,6 @@ spec: terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} initContainers: {{ tuple $envAll "airflow_server" $mounts_airflow_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} -{{- if .Values.prod_environment }} - - name: airflow-shipyard-init - image: {{ .Values.images.tags.shipyard }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/airflow-shipyard-init.sh - volumeMounts: - - name: airflow-bin - mountPath: /tmp/airflow-shipyard-init.sh - subPath: airflow-shipyard-init.sh - readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: /tmp/airflow/dags/ - - name: pod-shipyard-share-airflow-plugins - mountPath: /tmp/airflow/plugins/ -{{- end }} containers: - name: airflow-scheduler image: {{ .Values.images.tags.airflow }} @@ -88,21 +71,10 @@ spec: mountPath: {{ .Values.conf.airflow_config_file.path }} subPath: airflow.cfg readOnly: true -{{- if .Values.prod_environment }} - name: shipyard-etc mountPath: /usr/local/airflow/plugins/shipyard.conf subPath: shipyard.conf readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: pod-shipyard-share-airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{ else }} - - name: airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{ if $mounts_airflow_scheduler.volumeMounts }}{{ toYaml $mounts_airflow_scheduler.volumeMounts | indent 12 }}{{ end }} @@ -111,7 +83,6 @@ spec: configMap: name: airflow-etc defaultMode: 0444 -{{- if .Values.prod_environment }} - name: shipyard-etc configMap: name: shipyard-etc @@ -120,22 +91,7 @@ spec: configMap: name: airflow-bin defaultMode: 0555 - - name: pod-shipyard-share-airflow-dags - emptyDir: {} - - name: pod-shipyard-share-airflow-plugins - emptyDir: {} - name: airflow-logs emptyDir: {} -{{ else }} - - name: airflow-dags - hostPath: - path: {{ .Values.pod.mounts.dag_path }} - - name: airflow-plugins - hostPath: - path: {{ .Values.pod.mounts.plugin_path }} - - name: airflow-logs - hostPath: - path: {{ .Values.pod.mounts.log_path }} -{{- end }} {{ if $mounts_airflow_scheduler.volumes }}{{ toYaml $mounts_airflow_scheduler.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/charts/shipyard/templates/deployment-airflow-web.yaml b/charts/shipyard/templates/deployment-airflow-web.yaml index faaecb62..648f5564 100644 --- a/charts/shipyard/templates/deployment-airflow-web.yaml +++ b/charts/shipyard/templates/deployment-airflow-web.yaml @@ -44,23 +44,6 @@ spec: terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} initContainers: {{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} -{{- if .Values.prod_environment }} - - name: airflow-shipyard-init - image: {{ .Values.images.tags.shipyard }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/airflow-shipyard-init.sh - volumeMounts: - - name: airflow-bin - mountPath: /tmp/airflow-shipyard-init.sh - subPath: airflow-shipyard-init.sh - readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: /tmp/airflow/dags/ - - name: pod-shipyard-share-airflow-plugins - mountPath: /tmp/airflow/plugins/ -{{- end }} containers: - name: airflow-web image: {{ .Values.images.tags.airflow }} @@ -83,21 +66,10 @@ spec: mountPath: {{ .Values.conf.airflow_config_file.path }} subPath: airflow.cfg readOnly: true -{{- if .Values.prod_environment }} - name: shipyard-etc mountPath: /usr/local/airflow/plugins/shipyard.conf subPath: shipyard.conf readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: pod-shipyard-share-airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{ else }} - - name: airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }} @@ -106,7 +78,6 @@ spec: configMap: name: airflow-etc defaultMode: 0444 -{{- if .Values.prod_environment }} - name: shipyard-etc configMap: name: shipyard-etc @@ -115,22 +86,7 @@ spec: configMap: name: airflow-bin defaultMode: 0555 - - name: pod-shipyard-share-airflow-dags - emptyDir: {} - - name: pod-shipyard-share-airflow-plugins - emptyDir: {} - name: airflow-logs emptyDir: {} -{{ else }} - - name: airflow-dags - hostPath: - path: {{ .Values.pod.mounts.dag_path }} - - name: airflow-plugins - hostPath: - path: {{ .Values.pod.mounts.plugin_path }} - - name: airflow-logs - hostPath: - path: {{ .Values.pod.mounts.log_path }} -{{- end }} {{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml index a53473d7..c71b63eb 100644 --- a/charts/shipyard/templates/statefulset-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -79,22 +79,6 @@ spec: {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }} initContainers: {{ tuple $envAll "airflow_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} -{{- if .Values.prod_environment }} - - name: airflow-shipyard-init - image: {{ .Values.images.tags.shipyard }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/airflow-shipyard-init.sh - volumeMounts: - - name: airflow-bin - mountPath: /tmp/airflow-shipyard-init.sh - subPath: airflow-shipyard-init.sh - readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: /tmp/airflow/dags/ - - name: pod-shipyard-share-airflow-plugins - mountPath: /tmp/airflow/plugins/ - name: worker-perms image: {{ .Values.images.tags.airflow }} imagePullPolicy: {{ .Values.images.pull_policy }} @@ -109,7 +93,6 @@ spec: volumeMounts: - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{- end }} containers: - name: airflow-scheduler image: {{ .Values.images.tags.airflow }} @@ -128,21 +111,10 @@ spec: mountPath: {{ .Values.conf.airflow_config_file.path }} subPath: airflow.cfg readOnly: true -{{- if .Values.prod_environment }} - name: shipyard-etc mountPath: /usr/local/airflow/plugins/shipyard.conf subPath: shipyard.conf readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: pod-shipyard-share-airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{ else }} - - name: airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} - name: airflow-worker @@ -166,30 +138,12 @@ spec: mountPath: {{ .Values.conf.airflow_config_file.path }} subPath: airflow.cfg readOnly: true -{{- if .Values.prod_environment }} - name: shipyard-etc mountPath: /usr/local/airflow/plugins/shipyard.conf subPath: shipyard.conf readOnly: true - - name: pod-shipyard-share-airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: pod-shipyard-share-airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} -{{ else }} - - name: airflow-dags - mountPath: {{ .Values.conf.airflow.core.dags_folder }} - - name: airflow-plugins - mountPath: {{ .Values.conf.airflow.core.plugins_folder }} - - name: docker - mountPath: /var/run - readOnly: false - - name: pod-var-lib-docker - mountPath: /var/lib/docker - readOnly: false -{{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{- if .Values.prod_environment }} - name: airflow-logrotate image: {{ .Values.images.tags.airflow }} imagePullPolicy: {{ .Values.images.pull_policy }} @@ -214,13 +168,11 @@ spec: volumeMounts: - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{- end }} volumes: - name: airflow-etc configMap: name: airflow-etc defaultMode: 0444 -{{- if .Values.prod_environment }} - name: shipyard-etc configMap: name: shipyard-etc @@ -229,10 +181,6 @@ spec: configMap: name: airflow-bin defaultMode: 0555 - - name: pod-shipyard-share-airflow-dags - emptyDir: {} - - name: pod-shipyard-share-airflow-plugins - emptyDir: {} volumeClaimTemplates: - metadata: name: airflow-logs @@ -242,21 +190,4 @@ spec: requests: storage: {{ .Values.volume_worker.size }} storageClassName: {{ .Values.volume_worker.class_name }} -{{ else }} - - name: airflow-dags - hostPath: - path: {{ .Values.pod.mounts.dag_path }} - - name: airflow-plugins - hostPath: - path: {{ .Values.pod.mounts.plugin_path }} - - name: airflow-logs - hostPath: - path: {{ .Values.pod.mounts.log_path }} - - name: docker - hostPath: - path: /var/run - - name: pod-var-lib-docker - hostPath: - path: /var/lib/docker -{{- end }} {{- end }} diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 969ad4b3..998a4295 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -15,9 +15,6 @@ # This file provides defaults for shipyard and airflow -# Indicate whether it is production or development environment -prod_environment: true - labels: job: node_selector_key: ucp-control-plane @@ -701,9 +698,6 @@ conf: # End of Airflow config options pod: mounts: - dag_path: /home/ubuntu/workbench/dags - plugin_path: /home/ubuntu/workbench/plugins - log_path: /home/ubuntu/workbench/logs airflow_scheduler: # TODO: This is only used if the standalone scheduler is enabled. airflow_scheduler: diff --git a/images/airflow/Dockerfile b/images/airflow/Dockerfile index 8652f158..6b130962 100644 --- a/images/airflow/Dockerfile +++ b/images/airflow/Dockerfile @@ -104,7 +104,7 @@ ENV PBR_VERSION 0.1a1 # Shipyard # -# Shipyard provides core functionality used by the airflow plugins/operators +# Shipyard provides core functionality used by the Airflow plugins/operators # Since Shipyard and Airflow are built together as images, this should prevent # stale or out-of-date code between these parts. # Shipyard requirements, source and installation @@ -115,6 +115,20 @@ COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/ RUN cd /tmp/shipyard \ && python3 setup.py install +# Note: The value for the dags and plugins directories that are sourced +# from the values.yaml of the Shipyard Helm chart need to align with these +# directories. If they do not, airflow will not find the intended dags and +# plugins. +# +# Note: In the case of building images using the provided Makefile, a test is +# run against the built-in dags provided with Airflow. Since there is no Helm +# chart to reconfigure the airflow.cfg with these directories, these dags and +# plugins are not known to Airflow during the image test. +# +# Copy the plugins and dags that will be used by this Airflow image: +COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/ +COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/ + # Set work directory USER airflow WORKDIR ${AIRFLOW_HOME}