Browse Source

Build workflows into Airflow image

Changes to make the docker image build to include the workflows from
Shipyard, rather than adding them to the container during Helm install
of Shipyard. This also removes the "prod" switch, as it is now always
built the same way, with the workflows in place.

Change-Id: I4acd6195cbec32193e15621e75ccaeb9879455f5
Bryan Strassner 3 months ago
parent
commit
9725b0f337

+ 0
- 23
charts/shipyard/templates/bin/_airflow-shipyard-init.sh.tpl View File

@@ -1,23 +0,0 @@
1
-#!/bin/bash
2
-
3
-{{/*
4
-Copyright 2017 The Openstack-Helm Authors.
5
-Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
6
-
7
-Licensed under the Apache License, Version 2.0 (the "License");
8
-you may not use this file except in compliance with the License.
9
-You may obtain a copy of the License at
10
-
11
-   http://www.apache.org/licenses/LICENSE-2.0
12
-
13
-Unless required by applicable law or agreed to in writing, software
14
-distributed under the License is distributed on an "AS IS" BASIS,
15
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
-See the License for the specific language governing permissions and
17
-limitations under the License.
18
-*/}}
19
-
20
-set -ex
21
-
22
-cp -vaR /home/shipyard/shipyard/shipyard_airflow/dags/*.py /tmp/airflow/dags/
23
-cp -vaR /home/shipyard/shipyard/shipyard_airflow/plugins/*.py /tmp/airflow/plugins/

+ 0
- 2
charts/shipyard/templates/configmap-airflow-bin.yaml View File

@@ -27,6 +27,4 @@ data:
27 27
 {{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
28 28
   airflow-db-sync.sh: |+
29 29
 {{ tuple "bin/_airflow-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
30
-  airflow-shipyard-init.sh: |+
31
-{{ tuple "bin/_airflow-shipyard-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
32 30
 {{- end }}

+ 0
- 44
charts/shipyard/templates/deployment-airflow-scheduler.yaml View File

@@ -53,23 +53,6 @@ spec:
53 53
       terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
54 54
       initContainers:
55 55
 {{ tuple $envAll "airflow_server" $mounts_airflow_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
56
-{{- if .Values.prod_environment }}
57
-        - name: airflow-shipyard-init
58
-          image: {{ .Values.images.tags.shipyard }}
59
-          imagePullPolicy: {{ .Values.images.pull_policy }}
60
-{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
61
-          command:
62
-            - /tmp/airflow-shipyard-init.sh
63
-          volumeMounts:
64
-            - name: airflow-bin
65
-              mountPath: /tmp/airflow-shipyard-init.sh
66
-              subPath: airflow-shipyard-init.sh
67
-              readOnly: true
68
-            - name: pod-shipyard-share-airflow-dags
69
-              mountPath: /tmp/airflow/dags/
70
-            - name: pod-shipyard-share-airflow-plugins
71
-              mountPath: /tmp/airflow/plugins/
72
-{{- end }}
73 56
       containers:
74 57
         - name: airflow-scheduler
75 58
           image: {{ .Values.images.tags.airflow }}
@@ -88,21 +71,10 @@ spec:
88 71
               mountPath: {{ .Values.conf.airflow_config_file.path }}
89 72
               subPath: airflow.cfg
90 73
               readOnly: true
91
-{{- if .Values.prod_environment }}
92 74
             - name: shipyard-etc
93 75
               mountPath: /usr/local/airflow/plugins/shipyard.conf
94 76
               subPath: shipyard.conf
95 77
               readOnly: true
96
-            - name: pod-shipyard-share-airflow-dags
97
-              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
98
-            - name: pod-shipyard-share-airflow-plugins
99
-              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
100
-{{ else }}
101
-            - name: airflow-dags
102
-              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
103
-            - name: airflow-plugins
104
-              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
105
-{{- end }}
106 78
             - name: airflow-logs
107 79
               mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
108 80
 {{ if $mounts_airflow_scheduler.volumeMounts }}{{ toYaml $mounts_airflow_scheduler.volumeMounts | indent 12 }}{{ end }}
@@ -111,7 +83,6 @@ spec:
111 83
           configMap:
112 84
             name: airflow-etc
113 85
             defaultMode: 0444
114
-{{- if .Values.prod_environment }}
115 86
         - name: shipyard-etc
116 87
           configMap:
117 88
             name: shipyard-etc
@@ -120,22 +91,7 @@ spec:
120 91
           configMap:
121 92
             name: airflow-bin
122 93
             defaultMode: 0555
123
-        - name: pod-shipyard-share-airflow-dags
124
-          emptyDir: {}
125
-        - name: pod-shipyard-share-airflow-plugins
126
-          emptyDir: {}
127 94
         - name: airflow-logs
128 95
           emptyDir: {}
129
-{{ else }}
130
-        - name: airflow-dags
131
-          hostPath:
132
-            path: {{ .Values.pod.mounts.dag_path }}
133
-        - name: airflow-plugins
134
-          hostPath:
135
-            path: {{ .Values.pod.mounts.plugin_path }}
136
-        - name: airflow-logs
137
-          hostPath:
138
-            path: {{ .Values.pod.mounts.log_path }}
139
-{{- end }}
140 96
 {{ if $mounts_airflow_scheduler.volumes }}{{ toYaml $mounts_airflow_scheduler.volumes | indent 8 }}{{ end }}
141 97
 {{- end }}

+ 0
- 44
charts/shipyard/templates/deployment-airflow-web.yaml View File

@@ -44,23 +44,6 @@ spec:
44 44
       terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
45 45
       initContainers:
46 46
 {{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
47
-{{- if .Values.prod_environment }}
48
-        - name: airflow-shipyard-init
49
-          image: {{ .Values.images.tags.shipyard }}
50
-          imagePullPolicy: {{ .Values.images.pull_policy }}
51
-{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
52
-          command:
53
-            - /tmp/airflow-shipyard-init.sh
54
-          volumeMounts:
55
-            - name: airflow-bin
56
-              mountPath: /tmp/airflow-shipyard-init.sh
57
-              subPath: airflow-shipyard-init.sh
58
-              readOnly: true
59
-            - name: pod-shipyard-share-airflow-dags
60
-              mountPath: /tmp/airflow/dags/
61
-            - name: pod-shipyard-share-airflow-plugins
62
-              mountPath: /tmp/airflow/plugins/
63
-{{- end }}
64 47
       containers:
65 48
         - name: airflow-web
66 49
           image: {{ .Values.images.tags.airflow }}
@@ -83,21 +66,10 @@ spec:
83 66
               mountPath: {{ .Values.conf.airflow_config_file.path }}
84 67
               subPath: airflow.cfg
85 68
               readOnly: true
86
-{{- if .Values.prod_environment }}
87 69
             - name: shipyard-etc
88 70
               mountPath: /usr/local/airflow/plugins/shipyard.conf
89 71
               subPath: shipyard.conf
90 72
               readOnly: true
91
-            - name: pod-shipyard-share-airflow-dags
92
-              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
93
-            - name: pod-shipyard-share-airflow-plugins
94
-              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
95
-{{ else }}
96
-            - name: airflow-dags
97
-              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
98
-            - name: airflow-plugins
99
-              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
100
-{{- end }}
101 73
             - name: airflow-logs
102 74
               mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
103 75
 {{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }}
@@ -106,7 +78,6 @@ spec:
106 78
           configMap:
107 79
             name: airflow-etc
108 80
             defaultMode: 0444
109
-{{- if .Values.prod_environment }}
110 81
         - name: shipyard-etc
111 82
           configMap:
112 83
             name: shipyard-etc
@@ -115,22 +86,7 @@ spec:
115 86
           configMap:
116 87
             name: airflow-bin
117 88
             defaultMode: 0555
118
-        - name: pod-shipyard-share-airflow-dags
119
-          emptyDir: {}
120
-        - name: pod-shipyard-share-airflow-plugins
121
-          emptyDir: {}
122 89
         - name: airflow-logs
123 90
           emptyDir: {}
124
-{{ else }}
125
-        - name: airflow-dags
126
-          hostPath:
127
-            path: {{ .Values.pod.mounts.dag_path }}
128
-        - name: airflow-plugins
129
-          hostPath:
130
-            path: {{ .Values.pod.mounts.plugin_path }}
131
-        - name: airflow-logs
132
-          hostPath:
133
-            path: {{ .Values.pod.mounts.log_path }}
134
-{{- end }}
135 91
 {{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }}
136 92
 {{- end }}

+ 0
- 69
charts/shipyard/templates/statefulset-airflow-worker.yaml View File

@@ -79,22 +79,6 @@ spec:
79 79
         {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }}
80 80
       initContainers:
81 81
 {{ tuple $envAll "airflow_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
82
-{{- if .Values.prod_environment }}
83
-        - name: airflow-shipyard-init
84
-          image: {{ .Values.images.tags.shipyard }}
85
-          imagePullPolicy: {{ .Values.images.pull_policy }}
86
-{{ tuple $envAll $envAll.Values.pod.resources.shipyard_api | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
87
-          command:
88
-            - /tmp/airflow-shipyard-init.sh
89
-          volumeMounts:
90
-            - name: airflow-bin
91
-              mountPath: /tmp/airflow-shipyard-init.sh
92
-              subPath: airflow-shipyard-init.sh
93
-              readOnly: true
94
-            - name: pod-shipyard-share-airflow-dags
95
-              mountPath: /tmp/airflow/dags/
96
-            - name: pod-shipyard-share-airflow-plugins
97
-              mountPath: /tmp/airflow/plugins/
98 82
         - name: worker-perms
99 83
           image: {{ .Values.images.tags.airflow }}
100 84
           imagePullPolicy: {{ .Values.images.pull_policy }}
@@ -109,7 +93,6 @@ spec:
109 93
           volumeMounts:
110 94
             - name: airflow-logs
111 95
               mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
112
-{{- end }}
113 96
       containers:
114 97
         - name: airflow-scheduler
115 98
           image: {{ .Values.images.tags.airflow }}
@@ -128,21 +111,10 @@ spec:
128 111
               mountPath: {{ .Values.conf.airflow_config_file.path }}
129 112
               subPath: airflow.cfg
130 113
               readOnly: true
131
-{{- if .Values.prod_environment }}
132 114
             - name: shipyard-etc
133 115
               mountPath: /usr/local/airflow/plugins/shipyard.conf
134 116
               subPath: shipyard.conf
135 117
               readOnly: true
136
-            - name: pod-shipyard-share-airflow-dags
137
-              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
138
-            - name: pod-shipyard-share-airflow-plugins
139
-              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
140
-{{ else }}
141
-            - name: airflow-dags
142
-              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
143
-            - name: airflow-plugins
144
-              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
145
-{{- end }}
146 118
             - name: airflow-logs
147 119
               mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
148 120
         - name: airflow-worker
@@ -166,30 +138,12 @@ spec:
166 138
               mountPath: {{ .Values.conf.airflow_config_file.path }}
167 139
               subPath: airflow.cfg
168 140
               readOnly: true
169
-{{- if .Values.prod_environment }}
170 141
             - name: shipyard-etc
171 142
               mountPath: /usr/local/airflow/plugins/shipyard.conf
172 143
               subPath: shipyard.conf
173 144
               readOnly: true
174
-            - name: pod-shipyard-share-airflow-dags
175
-              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
176
-            - name: pod-shipyard-share-airflow-plugins
177
-              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
178
-{{ else }}
179
-            - name: airflow-dags
180
-              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
181
-            - name: airflow-plugins
182
-              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
183
-            - name: docker
184
-              mountPath: /var/run
185
-              readOnly: false
186
-            - name: pod-var-lib-docker
187
-              mountPath: /var/lib/docker
188
-              readOnly: false
189
-{{- end }}
190 145
             - name: airflow-logs
191 146
               mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
192
-{{- if .Values.prod_environment }}
193 147
         - name: airflow-logrotate
194 148
           image: {{ .Values.images.tags.airflow }}
195 149
           imagePullPolicy: {{ .Values.images.pull_policy }}
@@ -214,13 +168,11 @@ spec:
214 168
           volumeMounts:
215 169
             - name: airflow-logs
216 170
               mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
217
-{{- end }}
218 171
       volumes:
219 172
         - name: airflow-etc
220 173
           configMap:
221 174
             name: airflow-etc
222 175
             defaultMode: 0444
223
-{{- if .Values.prod_environment }}
224 176
         - name: shipyard-etc
225 177
           configMap:
226 178
             name: shipyard-etc
@@ -229,10 +181,6 @@ spec:
229 181
           configMap:
230 182
             name: airflow-bin
231 183
             defaultMode: 0555
232
-        - name: pod-shipyard-share-airflow-dags
233
-          emptyDir: {}
234
-        - name: pod-shipyard-share-airflow-plugins
235
-          emptyDir: {}
236 184
   volumeClaimTemplates:
237 185
   - metadata:
238 186
       name: airflow-logs
@@ -242,21 +190,4 @@ spec:
242 190
         requests:
243 191
           storage: {{ .Values.volume_worker.size }}
244 192
       storageClassName: {{ .Values.volume_worker.class_name }}
245
-{{ else }}
246
-        - name: airflow-dags
247
-          hostPath:
248
-            path: {{ .Values.pod.mounts.dag_path }}
249
-        - name: airflow-plugins
250
-          hostPath:
251
-            path: {{ .Values.pod.mounts.plugin_path }}
252
-        - name: airflow-logs
253
-          hostPath:
254
-            path: {{ .Values.pod.mounts.log_path }}
255
-        - name: docker
256
-          hostPath:
257
-            path: /var/run
258
-        - name: pod-var-lib-docker
259
-          hostPath:
260
-            path: /var/lib/docker
261
-{{- end }}
262 193
 {{- end }}

+ 0
- 6
charts/shipyard/values.yaml View File

@@ -15,9 +15,6 @@
15 15
 
16 16
 # This file provides defaults for shipyard and airflow
17 17
 
18
-# Indicate whether it is production or development environment
19
-prod_environment: true
20
-
21 18
 labels:
22 19
   job:
23 20
     node_selector_key: ucp-control-plane
@@ -701,9 +698,6 @@ conf:
701 698
     # End of Airflow config options
702 699
 pod:
703 700
   mounts:
704
-    dag_path: /home/ubuntu/workbench/dags
705
-    plugin_path: /home/ubuntu/workbench/plugins
706
-    log_path: /home/ubuntu/workbench/logs
707 701
     airflow_scheduler:
708 702
       # TODO: This is only used if the standalone scheduler is enabled.
709 703
       airflow_scheduler:

+ 15
- 1
images/airflow/Dockerfile View File

@@ -104,7 +104,7 @@ ENV PBR_VERSION 0.1a1
104 104
 
105 105
 # Shipyard
106 106
 #
107
-# Shipyard provides core functionality used by the airflow plugins/operators
107
+# Shipyard provides core functionality used by the Airflow plugins/operators
108 108
 # Since Shipyard and Airflow are built together as images, this should prevent
109 109
 # stale or out-of-date code between these parts.
110 110
 # Shipyard requirements, source and installation
@@ -115,6 +115,20 @@ COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
115 115
 RUN cd /tmp/shipyard \
116 116
     && python3 setup.py install
117 117
 
118
+# Note: The value for the dags and plugins directories that are sourced
119
+# from the values.yaml of the Shipyard Helm chart need to align with these
120
+# directories. If they do not, airflow will not find the intended dags and
121
+# plugins.
122
+#
123
+# Note: In the case of building images using the provided Makefile, a test is
124
+# run against the built-in dags provided with Airflow. Since there is no Helm
125
+# chart to reconfigure the airflow.cfg with these directories, these dags and
126
+# plugins are not known to Airflow during the image test.
127
+#
128
+# Copy the plugins and dags that will be used by this Airflow image:
129
+COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
130
+COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
131
+
118 132
 # Set work directory
119 133
 USER airflow
120 134
 WORKDIR ${AIRFLOW_HOME}

Loading…
Cancel
Save