Browse Source

Move airflow scheduler to worker statefulset

Moves the airflow sceduler to a container in the airflow-worker
statefulset so that its version lifecycle matches that of the worker.
Leaves the stand-alone scheduler in place to support upgradability from
prior installations that included a standalone scheduler. New
installations are advised to turn off the scheduler template from
rendering using the values.yaml flag.

This is an attempt to make disruptive upgrades to airflow less impactful
to a "update_site" action from Shipyard.

Additionally this removes the template for airflow-flower, which is not in use.

Change-Id: I0608793ee6aba1eb3ce0f5e9567655287014a0ca
Bryan Strassner 3 months ago
parent
commit
6b75c7119a

+ 0
- 74
charts/shipyard/templates/deployment-airflow-flower.yaml View File

@@ -1,74 +0,0 @@
1
-# Copyright 2017 The Openstack-Helm Authors.
2
-# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
3
-#
4
-# Licensed under the Apache License, Version 2.0 (the "License");
5
-# you may not use this file except in compliance with the License.
6
-# You may obtain a copy of the License at
7
-#
8
-#     http://www.apache.org/licenses/LICENSE-2.0
9
-#
10
-# Unless required by applicable law or agreed to in writing, software
11
-# distributed under the License is distributed on an "AS IS" BASIS,
12
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
-# See the License for the specific language governing permissions and
14
-# limitations under the License.
15
-
16
-{{- if .Values.manifests.deployment_airflow_flower }}
17
-{{- $envAll := . }}
18
-{{- $serviceAccountName := "airflow-flower" }}
19
-{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
20
-{{- $mounts_airflow_flower := .Values.pod.mounts.airflow_flower.airflow_flower }}
21
-{{- $mounts_airflow_flower_init := .Values.pod.mounts.airflow_flower.init_container }}
22
----
23
-apiVersion: apps/v1beta1
24
-kind: Deployment
25
-metadata:
26
-  name: airflow-flower
27
-  annotations:
28
-    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
29
-spec:
30
-  replicas: {{ .Values.pod.replicas.airflow.flower }}
31
-{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
32
-  template:
33
-    metadata:
34
-      labels:
35
-{{ tuple $envAll "airflow" "flower" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
36
-      annotations:
37
-        configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }}
38
-        configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }}
39
-    spec:
40
-      serviceAccountName: {{ $serviceAccountName }}
41
-      nodeSelector:
42
-        {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }}
43
-      restartPolicy: Always
44
-      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
45
-      initContainers:
46
-{{ tuple $envAll "airflow_server" $mounts_airflow_flower_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
47
-      containers:
48
-        - name: airflow-flower
49
-          image: {{ .Values.images.tags.airflow }}
50
-          imagePullPolicy: {{ .Values.images.pull_policy }}
51
-{{ tuple $envAll $envAll.Values.pod.resources.airflow.flower | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
52
-          env:
53
-          # To prevent the error: ValueError: invalid literal for int() with base 10: 'tcp://10.0.0.83:5555'
54
-          - name: FLOWER_PORT
55
-            value: "5555"
56
-          ports:
57
-            - containerPort: {{ .Values.network.airflow.flower.port }}
58
-          args: ["flower"]
59
-          readinessProbe:
60
-            tcpSocket:
61
-              port: {{ .Values.network.airflow.flower.port }}
62
-          volumeMounts:
63
-            - name: airflow-etc
64
-              mountPath: {{ .Values.conf.airflow_config_file.path }}
65
-              subPath: airflow.cfg
66
-              readOnly: true
67
-{{ if $mounts_airflow_flower.volumeMounts }}{{ toYaml $mounts_airflow_flower.volumeMounts | indent 12 }}{{ end }}
68
-      volumes:
69
-        - name: airflow-etc
70
-          configMap:
71
-            name: airflow-etc
72
-            defaultMode: 0444
73
-{{ if $mounts_airflow_flower.volumes }}{{ toYaml $mounts_airflow_flower.volumes | indent 8 }}{{ end }}
74
-{{- end }}

+ 9
- 0
charts/shipyard/templates/deployment-airflow-scheduler.yaml View File

@@ -13,6 +13,15 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
+# NOTE: This is a "standalone" scheduler definition.
17
+# TODO: Remove this template after it is no longer needed. Until the worker
18
+#   statefulset contains the scheduler, this scheduler pod needs to be
19
+#   available to provide continuity of a software/site/update. Once
20
+#   Shipyard/Airflow is running with the Scheduler/Worker combo, this
21
+#   scheduler pod can be disabled by turning off the
22
+#   manifests.deployment_airflow_scheduler value.
23
+#
24
+
16 25
 {{- if .Values.manifests.deployment_airflow_scheduler }}
17 26
 {{- $envAll := . }}
18 27
 {{- $serviceAccountName := "airflow-scheduler" }}

+ 0
- 44
charts/shipyard/templates/service-airflow-flower.yaml View File

@@ -1,44 +0,0 @@
1
-{{/*
2
-Copyright 2017 The Openstack-Helm Authors.
3
-Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
4
-
5
-Licensed under the Apache License, Version 2.0 (the "License");
6
-you may not use this file except in compliance with the License.
7
-You may obtain a copy of the License at
8
-
9
-   http://www.apache.org/licenses/LICENSE-2.0
10
-
11
-Unless required by applicable law or agreed to in writing, software
12
-distributed under the License is distributed on an "AS IS" BASIS,
13
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
-See the License for the specific language governing permissions and
15
-limitations under the License.
16
-*/}}
17
-
18
-{{- if .Values.manifests.service_airflow_flower }}
19
-{{- $envAll := . }}
20
----
21
-apiVersion: v1
22
-kind: Service
23
-metadata:
24
-  name: {{ tuple "airflow_flower" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
25
-spec:
26
-  ports:
27
-    {{ if .Values.network.airflow.flower.enable_node_port }}
28
-    - name: http
29
-      nodePort: {{ .Values.network.airflow.flower.node_port }}
30
-      port: {{ .Values.network.airflow.flower.port }}
31
-      protocol: TCP
32
-      targetPort: {{ .Values.network.airflow.flower.port }}
33
-    {{ else }}
34
-    - name: http
35
-      port: {{ .Values.network.airflow.flower.port }}
36
-      protocol: TCP
37
-      targetPort: {{ .Values.network.airflow.flower.port }}
38
-    {{ end }}
39
-  selector:
40
-{{ tuple $envAll "airflow" "flower" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
41
-  {{ if .Values.network.airflow.flower.enable_node_port }}
42
-  type: NodePort
43
-  {{ end }}
44
-{{- end }}

+ 34
- 0
charts/shipyard/templates/statefulset-airflow-worker.yaml View File

@@ -111,6 +111,40 @@ spec:
111 111
               mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
112 112
 {{- end }}
113 113
       containers:
114
+        - name: airflow-scheduler
115
+          image: {{ .Values.images.tags.airflow }}
116
+          imagePullPolicy: {{ .Values.images.pull_policy }}
117
+{{ tuple $envAll $envAll.Values.pod.resources.airflow.scheduler | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
118
+          env:
119
+          - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB
120
+            valueFrom:
121
+                secretKeyRef:
122
+                    name: {{ .Values.secrets.postgresql_airflow_db.user }}
123
+                    key: AIRFLOW_DATABASE_URI
124
+          # Set to -1 to stop scheduler from going into crash loops
125
+          args: ["scheduler", "-n", "-1" ]
126
+          volumeMounts:
127
+            - name: airflow-etc
128
+              mountPath: {{ .Values.conf.airflow_config_file.path }}
129
+              subPath: airflow.cfg
130
+              readOnly: true
131
+{{- if .Values.prod_environment }}
132
+            - name: shipyard-etc
133
+              mountPath: /usr/local/airflow/plugins/shipyard.conf
134
+              subPath: shipyard.conf
135
+              readOnly: true
136
+            - name: pod-shipyard-share-airflow-dags
137
+              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
138
+            - name: pod-shipyard-share-airflow-plugins
139
+              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
140
+{{ else }}
141
+            - name: airflow-dags
142
+              mountPath: {{ .Values.conf.airflow.core.dags_folder }}
143
+            - name: airflow-plugins
144
+              mountPath: {{ .Values.conf.airflow.core.plugins_folder }}
145
+{{- end }}
146
+            - name: airflow-logs
147
+              mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
114 148
         - name: airflow-worker
115 149
           image: {{ .Values.images.tags.airflow }}
116 150
           imagePullPolicy: {{ .Values.images.pull_policy }}

+ 5
- 35
charts/shipyard/values.yaml View File

@@ -71,10 +71,6 @@ network:
71 71
     ingress:
72 72
       public: true
73 73
       proxy_read_timeout: 600
74
-    flower:
75
-      name: airflow-flower
76
-      port: 5555
77
-      enable_node_port: false
78 74
     web:
79 75
       name: airflow-web
80 76
       port: 8080
@@ -94,8 +90,6 @@ dependencies:
94 90
       services:
95 91
         - service: postgresql_shipyard_db
96 92
           endpoint: internal
97
-        - service: airflow_flower
98
-          endpoint: internal
99 93
         - service: airflow_web
100 94
           endpoint: internal
101 95
     shipyard_db_sync:
@@ -140,8 +134,6 @@ dependencies:
140 134
         - shipyard-ks-user
141 135
         - shipyard-ks-endpoints
142 136
       services:
143
-        - service: airflow_flower
144
-          endpoint: internal
145 137
         - service: airflow_web
146 138
           endpoint: internal
147 139
         - service: identity
@@ -239,19 +231,6 @@ endpoints:
239 231
       default: http
240 232
     host_fqdn_override:
241 233
       default: null
242
-  airflow_flower:
243
-    name: airflow-flower
244
-    hosts:
245
-      default: airflow-flower
246
-    port:
247
-      airflow_flower:
248
-        default: 5555
249
-    path:
250
-     default: /
251
-    scheme:
252
-      default: http
253
-    host_fqdn_override:
254
-      default: null
255 234
   airflow_worker:
256 235
     name: airflow-worker
257 236
     hosts:
@@ -725,15 +704,14 @@ pod:
725 704
     dag_path: /home/ubuntu/workbench/dags
726 705
     plugin_path: /home/ubuntu/workbench/plugins
727 706
     log_path: /home/ubuntu/workbench/logs
728
-    airflow_flower:
729
-      init_container: null
730
-      airflow_flower:
731 707
     airflow_scheduler:
732
-      init_container: null
708
+      # TODO: This is only used if the standalone scheduler is enabled.
733 709
       airflow_scheduler:
710
+      init_container: null
734 711
     airflow_worker:
735 712
       init_container: null
736 713
       airflow_worker:
714
+      airflow_scheduler:
737 715
     airflow_web:
738 716
       init_container: null
739 717
       airflow_web:
@@ -752,7 +730,6 @@ pod:
752 730
     airflow:
753 731
       web: 2
754 732
       worker: 2
755
-      flower: 2
756 733
       scheduler: 2
757 734
   lifecycle:
758 735
     upgrades:
@@ -770,13 +747,6 @@ pod:
770 747
   resources:
771 748
     enabled: false
772 749
     airflow:
773
-      flower:
774
-        limits:
775
-          memory: "128Mi"
776
-          cpu: "100m"
777
-        requests:
778
-          memory: "128Mi"
779
-          cpu: "100m"
780 750
       logrotate:
781 751
         limits:
782 752
           memory: "128Mi"
@@ -890,7 +860,8 @@ manifests:
890 860
   configmap_shipyard_etc: true
891 861
   configmap_airflow_bin: true
892 862
   configmap_airflow_etc: true
893
-  deployment_airflow_flower: true
863
+  # TODO: Set this to false only if a new deployment, or if the worker pod is
864
+  #   running the scheduler
894 865
   deployment_airflow_scheduler: true
895 866
   deployment_shipyard: true
896 867
   deployment_airflow_web: true
@@ -911,7 +882,6 @@ manifests:
911 882
   secret_keystone: true
912 883
   secret_rabbitmq: true
913 884
   service_airflow_ingress: true
914
-  service_airflow_flower: true
915 885
   service_shipyard: true
916 886
   service_shipyard_ingress: true
917 887
   service_airflow_web: true

+ 26
- 55
doc/source/deployment-guide.rst View File

@@ -39,68 +39,39 @@ developers to bring up Shipyard and the rest of the Airship components on a
39 39
 single Ubuntu Virtual Machine.
40 40
 
41 41
 The deployment is fully automated and can take a while to complete. It can take
42
-30 minutes to an hour for a full deployment to complete.
42
+30 minutes to an hour or more for a full deployment to complete.
43 43
 
44 44
 Post Deployment
45 45
 ---------------
46 46
 
47
-#. The environment should resemble the following after executing the required
47
+#. The environment should include the following after executing the required
48 48
    steps::
49 49
 
50
-    # sudo kubectl get pods -n ucp
50
+    # sudo kubectl get pods -n ucp | grep -v Completed
51 51
     NAME                                   READY     STATUS    RESTARTS   AGE
52
-    airflow-flower-6cdc6f9cb4-5r62v        1/1       Running   0          3h
53
-    airflow-scheduler-6d54445bf8-6ldrd     1/1       Running   0          3h
54
-    airflow-web-7bd69d857d-qlptj           1/1       Running   0          3h
55
-    airflow-worker-666696d6c5-vffpg        1/1       Running   0          3h
56
-    armada-api-84df5b7fc9-4nxp5            1/1       Running   0          4h
57
-    barbican-api-85c956c84f-p4q7h          1/1       Running   0          4h
58
-    deckhand-5468d59455-2mcqd              1/1       Running   0          4h
59
-    drydock-api-f9897cf44-csbc8            1/1       Running   0          4h
60
-    drydock-api-f9897cf44-jgv4q            1/1       Running   0          4h
61
-    etcd-5bcbbd679c-rb5rf                  1/1       Running   0          4h
62
-    ingress-api-xvkzx                      1/1       Running   0          4h
63
-    ingress-error-pages-5d79688f6c-9b8xc   1/1       Running   0          4h
64
-    keystone-api-6bc85c98-886mg            1/1       Running   0          4h
65
-    maas-rack-5d4b84c4d5-dt87j             1/1       Running   0          4h
66
-    maas-region-0                          1/1       Running   0          4h
67
-    mariadb-0                              1/1       Running   0          4h
68
-    mariadb-1                              1/1       Running   0          4h
69
-    mariadb-2                              1/1       Running   0          4h
70
-    memcached-5bf49657db-kq6qh             1/1       Running   0          4h
71
-    postgresql-0                           1/1       Running   0          4h
72
-    rabbitmq-f68649644-pnw6p               1/1       Running   0          4h
73
-    shipyard-6f4c7765d-n2kx6               1/1       Running   0          3h
74
-
75
-#. Check that all relevant helm charts have been deployed::
76
-
77
-    # sudo helm ls
78
-    NAME                                    REVISION        UPDATED                         STATUS          CHART                           NAMESPACE
79
-    ucp-armada                              1               Fri Dec  1 10:03:44 2017        DEPLOYED        armada-0.1.0                    ucp
80
-    ucp-barbican                            1               Fri Dec  1 10:03:47 2017        DEPLOYED        barbican-0.1.0                  ucp
81
-    ucp-calico                              1               Fri Dec  1 10:00:05 2017        DEPLOYED        calico-0.1.0                    kube-system
82
-    ucp-calico-etcd                         1               Fri Dec  1 09:59:28 2017        DEPLOYED        etcd-0.1.0                      kube-system
83
-    ucp-ceph                                1               Fri Dec  1 10:00:58 2017        DEPLOYED        ceph-0.1.0                      ceph
84
-    ucp-coredns                             1               Fri Dec  1 10:00:26 2017        DEPLOYED        coredns-0.1.0                   kube-system
85
-    ucp-deckhand                            1               Fri Dec  1 10:03:39 2017        DEPLOYED        deckhand-0.1.0                  ucp
86
-    ucp-drydock                             1               Fri Dec  1 10:03:37 2017        DEPLOYED        drydock-0.1.0                   ucp
87
-    ucp-etcd-rabbitmq                       1               Fri Dec  1 10:02:44 2017        DEPLOYED        etcd-0.1.0                      ucp
88
-    ucp-ingress                             1               Fri Dec  1 10:02:45 2017        DEPLOYED        ingress-0.1.0                   ucp
89
-    ucp-keystone                            1               Fri Dec  1 10:03:45 2017        DEPLOYED        keystone-0.1.0                  ucp
90
-    ucp-kubernetes-apiserver                1               Fri Dec  1 10:00:32 2017        DEPLOYED        apiserver-0.1.0                 kube-system
91
-    ucp-kubernetes-controller-manager       1               Fri Dec  1 10:00:33 2017        DEPLOYED        controller_manager-0.1.0        kube-system
92
-    ucp-kubernetes-etcd                     1               Fri Dec  1 10:00:31 2017        DEPLOYED        etcd-0.1.0                      kube-system
93
-    ucp-kubernetes-proxy                    1               Fri Dec  1 09:58:46 2017        DEPLOYED        proxy-0.1.0                     kube-system
94
-    ucp-kubernetes-scheduler                1               Fri Dec  1 10:00:34 2017        DEPLOYED        scheduler-0.1.0                 kube-system
95
-    ucp-maas                                1               Fri Dec  1 10:03:36 2017        DEPLOYED        maas-0.1.0                      ucp
96
-    ucp-maas-postgresql                     1               Fri Dec  1 10:02:44 2017        DEPLOYED        postgresql-0.1.0                ucp
97
-    ucp-rabbitmq                            1               Fri Dec  1 10:02:45 2017        DEPLOYED        rabbitmq-0.1.0                  ucp
98
-    ucp-rbac                                1               Fri Dec  1 10:00:44 2017        DEPLOYED        rbac-0.1.0                      kube-system
99
-    ucp-shipyard                            1               Fri Dec  1 10:38:08 2017        DEPLOYED        shipyard-0.1.0                  ucp
100
-    ucp-ucp-ceph-config                     1               Fri Dec  1 10:02:40 2017        DEPLOYED        ceph-0.1.0                      ucp
101
-    ucp-ucp-mariadb                         1               Fri Dec  1 10:02:43 2017        DEPLOYED        mariadb-0.1.0                   ucp
102
-    ucp-ucp-memcached                       1               Fri Dec  1 10:02:44 2017        DEPLOYED        memcached-0.1.0                 ucp
103
-
52
+    airflow-scheduler-79754bfdd5-2wpxn     1/1       Running   0          4m
53
+    airflow-web-7679866685-g99qm           1/1       Running   0          4m
54
+    airflow-worker-0                       3/3       Running   0          4m
55
+    airship-ucp-keystone-memcached-mem...  1/1       Running   0          31m
56
+    airship-ucp-rabbitmq-rabbitmq-0        1/1       Running   0          35m
57
+    armada-api-5488cbdb99-zjb8n            1/1       Running   0          12m
58
+    barbican-api-5fc8f7d6f-s7h7j           1/1       Running   0          11m
59
+    deckhand-api-7b476d6c46-qlvtm          1/1       Running   0          8m
60
+    drydock-api-5f9fdc858d-lnxvj           1/1       Running   0          1m
61
+    ingress-6cd5b89d5d-hzfzj               1/1       Running   0          35m
62
+    ingress-error-pages-5c97bb46bb-zqqbx   1/1       Running   0          35m
63
+    keystone-api-7657986b8c-6bf92          1/1       Running   0          31m
64
+    maas-ingress-66447d7445-mgklj          2/2       Running   0          27m
65
+    maas-ingress-errors-8686d56d98-vrjzg   1/1       Running   0          27m
66
+    maas-rack-0                            1/1       Running   0          27m
67
+    maas-region-0                          2/2       Running   0          27m
68
+    mariadb-ingress-6c4f9c76f-lk9ff        1/1       Running   0          35m
69
+    mariadb-ingress-6c4f9c76f-ns5kj        1/1       Running   0          35m
70
+    mariadb-ingress-error-pages-5dd6fb...  1/1       Running   0          35m
71
+    mariadb-server-0                       1/1       Running   0          35m
72
+    postgresql-0                           1/1       Running   0          32m
73
+    promenade-api-764b765d77-ffhv4         1/1       Running   0          7m
74
+    shipyard-api-69888d9f68-8ljfk          1/1       Running   0          4m
104 75
 
105 76
 .. _airship-in-a-bottle: https://git.airshipit.org/cgit/airship-in-a-bottle
106 77
 .. _dev_minimal: https://git.airshipit.org/cgit/airship-in-a-bottle/tree/manifests/dev_minimal

Loading…
Cancel
Save