diff --git a/charts/shipyard/templates/deployment-airflow-flower.yaml b/charts/shipyard/templates/deployment-airflow-flower.yaml deleted file mode 100644 index 8efecfae..00000000 --- a/charts/shipyard/templates/deployment-airflow-flower.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.manifests.deployment_airflow_flower }} -{{- $envAll := . }} -{{- $serviceAccountName := "airflow-flower" }} -{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} -{{- $mounts_airflow_flower := .Values.pod.mounts.airflow_flower.airflow_flower }} -{{- $mounts_airflow_flower_init := .Values.pod.mounts.airflow_flower.init_container }} ---- -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: airflow-flower - annotations: - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} -spec: - replicas: {{ .Values.pod.replicas.airflow.flower }} -{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} - template: - metadata: - labels: -{{ tuple $envAll "airflow" "flower" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: - configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }} - spec: - serviceAccountName: {{ $serviceAccountName }} - nodeSelector: - {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }} - restartPolicy: Always - terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} - initContainers: -{{ tuple $envAll "airflow_server" $mounts_airflow_flower_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: airflow-flower - image: {{ .Values.images.tags.airflow }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.airflow.flower | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - env: - # To prevent the error: ValueError: invalid literal for int() with base 10: 'tcp://10.0.0.83:5555' - - name: FLOWER_PORT - value: "5555" - ports: - - containerPort: {{ .Values.network.airflow.flower.port }} - args: ["flower"] - readinessProbe: - tcpSocket: - port: {{ .Values.network.airflow.flower.port }} - volumeMounts: - - name: airflow-etc - mountPath: {{ .Values.conf.airflow_config_file.path }} - subPath: airflow.cfg - readOnly: true -{{ if $mounts_airflow_flower.volumeMounts }}{{ toYaml $mounts_airflow_flower.volumeMounts | indent 12 }}{{ end }} - volumes: - - name: airflow-etc - configMap: - name: airflow-etc - defaultMode: 0444 -{{ if $mounts_airflow_flower.volumes }}{{ toYaml $mounts_airflow_flower.volumes | indent 8 }}{{ end }} -{{- end }} diff --git a/charts/shipyard/templates/deployment-airflow-scheduler.yaml b/charts/shipyard/templates/deployment-airflow-scheduler.yaml index f38b5021..8c6c597f 100644 --- a/charts/shipyard/templates/deployment-airflow-scheduler.yaml +++ b/charts/shipyard/templates/deployment-airflow-scheduler.yaml @@ -13,6 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# NOTE: This is a "standalone" scheduler definition. +# TODO: Remove this template after it is no longer needed. Until the worker +# statefulset contains the scheduler, this scheduler pod needs to be +# available to provide continuity of a software/site/update. Once +# Shipyard/Airflow is running with the Scheduler/Worker combo, this +# scheduler pod can be disabled by turning off the +# manifests.deployment_airflow_scheduler value. +# + {{- if .Values.manifests.deployment_airflow_scheduler }} {{- $envAll := . }} {{- $serviceAccountName := "airflow-scheduler" }} diff --git a/charts/shipyard/templates/service-airflow-flower.yaml b/charts/shipyard/templates/service-airflow-flower.yaml deleted file mode 100644 index e7edab39..00000000 --- a/charts/shipyard/templates/service-airflow-flower.yaml +++ /dev/null @@ -1,44 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.service_airflow_flower }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "airflow_flower" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -spec: - ports: - {{ if .Values.network.airflow.flower.enable_node_port }} - - name: http - nodePort: {{ .Values.network.airflow.flower.node_port }} - port: {{ .Values.network.airflow.flower.port }} - protocol: TCP - targetPort: {{ .Values.network.airflow.flower.port }} - {{ else }} - - name: http - port: {{ .Values.network.airflow.flower.port }} - protocol: TCP - targetPort: {{ .Values.network.airflow.flower.port }} - {{ end }} - selector: -{{ tuple $envAll "airflow" "flower" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - {{ if .Values.network.airflow.flower.enable_node_port }} - type: NodePort - {{ end }} -{{- end }} diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml index 9cf6c33c..a53473d7 100644 --- a/charts/shipyard/templates/statefulset-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -111,6 +111,40 @@ spec: mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{- end }} containers: + - name: airflow-scheduler + image: {{ .Values.images.tags.airflow }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.airflow.scheduler | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI + # Set to -1 to stop scheduler from going into crash loops + args: ["scheduler", "-n", "-1" ] + volumeMounts: + - name: airflow-etc + mountPath: {{ .Values.conf.airflow_config_file.path }} + subPath: airflow.cfg + readOnly: true +{{- if .Values.prod_environment }} + - name: shipyard-etc + mountPath: /usr/local/airflow/plugins/shipyard.conf + subPath: shipyard.conf + readOnly: true + - name: pod-shipyard-share-airflow-dags + mountPath: {{ .Values.conf.airflow.core.dags_folder }} + - name: pod-shipyard-share-airflow-plugins + mountPath: {{ .Values.conf.airflow.core.plugins_folder }} +{{ else }} + - name: airflow-dags + mountPath: {{ .Values.conf.airflow.core.dags_folder }} + - name: airflow-plugins + mountPath: {{ .Values.conf.airflow.core.plugins_folder }} +{{- end }} + - name: airflow-logs + mountPath: {{ .Values.conf.airflow.core.base_log_folder }} - name: airflow-worker image: {{ .Values.images.tags.airflow }} imagePullPolicy: {{ .Values.images.pull_policy }} diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 9885db4b..969ad4b3 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -71,10 +71,6 @@ network: ingress: public: true proxy_read_timeout: 600 - flower: - name: airflow-flower - port: 5555 - enable_node_port: false web: name: airflow-web port: 8080 @@ -94,8 +90,6 @@ dependencies: services: - service: postgresql_shipyard_db endpoint: internal - - service: airflow_flower - endpoint: internal - service: airflow_web endpoint: internal shipyard_db_sync: @@ -140,8 +134,6 @@ dependencies: - shipyard-ks-user - shipyard-ks-endpoints services: - - service: airflow_flower - endpoint: internal - service: airflow_web endpoint: internal - service: identity @@ -239,19 +231,6 @@ endpoints: default: http host_fqdn_override: default: null - airflow_flower: - name: airflow-flower - hosts: - default: airflow-flower - port: - airflow_flower: - default: 5555 - path: - default: / - scheme: - default: http - host_fqdn_override: - default: null airflow_worker: name: airflow-worker hosts: @@ -725,15 +704,14 @@ pod: dag_path: /home/ubuntu/workbench/dags plugin_path: /home/ubuntu/workbench/plugins log_path: /home/ubuntu/workbench/logs - airflow_flower: - init_container: null - airflow_flower: airflow_scheduler: - init_container: null + # TODO: This is only used if the standalone scheduler is enabled. airflow_scheduler: + init_container: null airflow_worker: init_container: null airflow_worker: + airflow_scheduler: airflow_web: init_container: null airflow_web: @@ -752,7 +730,6 @@ pod: airflow: web: 2 worker: 2 - flower: 2 scheduler: 2 lifecycle: upgrades: @@ -770,13 +747,6 @@ pod: resources: enabled: false airflow: - flower: - limits: - memory: "128Mi" - cpu: "100m" - requests: - memory: "128Mi" - cpu: "100m" logrotate: limits: memory: "128Mi" @@ -890,7 +860,8 @@ manifests: configmap_shipyard_etc: true configmap_airflow_bin: true configmap_airflow_etc: true - deployment_airflow_flower: true + # TODO: Set this to false only if a new deployment, or if the worker pod is + # running the scheduler deployment_airflow_scheduler: true deployment_shipyard: true deployment_airflow_web: true @@ -911,7 +882,6 @@ manifests: secret_keystone: true secret_rabbitmq: true service_airflow_ingress: true - service_airflow_flower: true service_shipyard: true service_shipyard_ingress: true service_airflow_web: true diff --git a/doc/source/deployment-guide.rst b/doc/source/deployment-guide.rst index 7367bfff..6251bb51 100644 --- a/doc/source/deployment-guide.rst +++ b/doc/source/deployment-guide.rst @@ -39,68 +39,39 @@ developers to bring up Shipyard and the rest of the Airship components on a single Ubuntu Virtual Machine. The deployment is fully automated and can take a while to complete. It can take -30 minutes to an hour for a full deployment to complete. +30 minutes to an hour or more for a full deployment to complete. Post Deployment --------------- -#. The environment should resemble the following after executing the required +#. The environment should include the following after executing the required steps:: - # sudo kubectl get pods -n ucp + # sudo kubectl get pods -n ucp | grep -v Completed NAME READY STATUS RESTARTS AGE - airflow-flower-6cdc6f9cb4-5r62v 1/1 Running 0 3h - airflow-scheduler-6d54445bf8-6ldrd 1/1 Running 0 3h - airflow-web-7bd69d857d-qlptj 1/1 Running 0 3h - airflow-worker-666696d6c5-vffpg 1/1 Running 0 3h - armada-api-84df5b7fc9-4nxp5 1/1 Running 0 4h - barbican-api-85c956c84f-p4q7h 1/1 Running 0 4h - deckhand-5468d59455-2mcqd 1/1 Running 0 4h - drydock-api-f9897cf44-csbc8 1/1 Running 0 4h - drydock-api-f9897cf44-jgv4q 1/1 Running 0 4h - etcd-5bcbbd679c-rb5rf 1/1 Running 0 4h - ingress-api-xvkzx 1/1 Running 0 4h - ingress-error-pages-5d79688f6c-9b8xc 1/1 Running 0 4h - keystone-api-6bc85c98-886mg 1/1 Running 0 4h - maas-rack-5d4b84c4d5-dt87j 1/1 Running 0 4h - maas-region-0 1/1 Running 0 4h - mariadb-0 1/1 Running 0 4h - mariadb-1 1/1 Running 0 4h - mariadb-2 1/1 Running 0 4h - memcached-5bf49657db-kq6qh 1/1 Running 0 4h - postgresql-0 1/1 Running 0 4h - rabbitmq-f68649644-pnw6p 1/1 Running 0 4h - shipyard-6f4c7765d-n2kx6 1/1 Running 0 3h - -#. Check that all relevant helm charts have been deployed:: - - # sudo helm ls - NAME REVISION UPDATED STATUS CHART NAMESPACE - ucp-armada 1 Fri Dec 1 10:03:44 2017 DEPLOYED armada-0.1.0 ucp - ucp-barbican 1 Fri Dec 1 10:03:47 2017 DEPLOYED barbican-0.1.0 ucp - ucp-calico 1 Fri Dec 1 10:00:05 2017 DEPLOYED calico-0.1.0 kube-system - ucp-calico-etcd 1 Fri Dec 1 09:59:28 2017 DEPLOYED etcd-0.1.0 kube-system - ucp-ceph 1 Fri Dec 1 10:00:58 2017 DEPLOYED ceph-0.1.0 ceph - ucp-coredns 1 Fri Dec 1 10:00:26 2017 DEPLOYED coredns-0.1.0 kube-system - ucp-deckhand 1 Fri Dec 1 10:03:39 2017 DEPLOYED deckhand-0.1.0 ucp - ucp-drydock 1 Fri Dec 1 10:03:37 2017 DEPLOYED drydock-0.1.0 ucp - ucp-etcd-rabbitmq 1 Fri Dec 1 10:02:44 2017 DEPLOYED etcd-0.1.0 ucp - ucp-ingress 1 Fri Dec 1 10:02:45 2017 DEPLOYED ingress-0.1.0 ucp - ucp-keystone 1 Fri Dec 1 10:03:45 2017 DEPLOYED keystone-0.1.0 ucp - ucp-kubernetes-apiserver 1 Fri Dec 1 10:00:32 2017 DEPLOYED apiserver-0.1.0 kube-system - ucp-kubernetes-controller-manager 1 Fri Dec 1 10:00:33 2017 DEPLOYED controller_manager-0.1.0 kube-system - ucp-kubernetes-etcd 1 Fri Dec 1 10:00:31 2017 DEPLOYED etcd-0.1.0 kube-system - ucp-kubernetes-proxy 1 Fri Dec 1 09:58:46 2017 DEPLOYED proxy-0.1.0 kube-system - ucp-kubernetes-scheduler 1 Fri Dec 1 10:00:34 2017 DEPLOYED scheduler-0.1.0 kube-system - ucp-maas 1 Fri Dec 1 10:03:36 2017 DEPLOYED maas-0.1.0 ucp - ucp-maas-postgresql 1 Fri Dec 1 10:02:44 2017 DEPLOYED postgresql-0.1.0 ucp - ucp-rabbitmq 1 Fri Dec 1 10:02:45 2017 DEPLOYED rabbitmq-0.1.0 ucp - ucp-rbac 1 Fri Dec 1 10:00:44 2017 DEPLOYED rbac-0.1.0 kube-system - ucp-shipyard 1 Fri Dec 1 10:38:08 2017 DEPLOYED shipyard-0.1.0 ucp - ucp-ucp-ceph-config 1 Fri Dec 1 10:02:40 2017 DEPLOYED ceph-0.1.0 ucp - ucp-ucp-mariadb 1 Fri Dec 1 10:02:43 2017 DEPLOYED mariadb-0.1.0 ucp - ucp-ucp-memcached 1 Fri Dec 1 10:02:44 2017 DEPLOYED memcached-0.1.0 ucp - + airflow-scheduler-79754bfdd5-2wpxn 1/1 Running 0 4m + airflow-web-7679866685-g99qm 1/1 Running 0 4m + airflow-worker-0 3/3 Running 0 4m + airship-ucp-keystone-memcached-mem... 1/1 Running 0 31m + airship-ucp-rabbitmq-rabbitmq-0 1/1 Running 0 35m + armada-api-5488cbdb99-zjb8n 1/1 Running 0 12m + barbican-api-5fc8f7d6f-s7h7j 1/1 Running 0 11m + deckhand-api-7b476d6c46-qlvtm 1/1 Running 0 8m + drydock-api-5f9fdc858d-lnxvj 1/1 Running 0 1m + ingress-6cd5b89d5d-hzfzj 1/1 Running 0 35m + ingress-error-pages-5c97bb46bb-zqqbx 1/1 Running 0 35m + keystone-api-7657986b8c-6bf92 1/1 Running 0 31m + maas-ingress-66447d7445-mgklj 2/2 Running 0 27m + maas-ingress-errors-8686d56d98-vrjzg 1/1 Running 0 27m + maas-rack-0 1/1 Running 0 27m + maas-region-0 2/2 Running 0 27m + mariadb-ingress-6c4f9c76f-lk9ff 1/1 Running 0 35m + mariadb-ingress-6c4f9c76f-ns5kj 1/1 Running 0 35m + mariadb-ingress-error-pages-5dd6fb... 1/1 Running 0 35m + mariadb-server-0 1/1 Running 0 35m + postgresql-0 1/1 Running 0 32m + promenade-api-764b765d77-ffhv4 1/1 Running 0 7m + shipyard-api-69888d9f68-8ljfk 1/1 Running 0 4m .. _airship-in-a-bottle: https://git.airshipit.org/cgit/airship-in-a-bottle .. _dev_minimal: https://git.airshipit.org/cgit/airship-in-a-bottle/tree/manifests/dev_minimal