diff --git a/charts/shipyard/templates/deployment-airflow-scheduler.yaml b/charts/shipyard/templates/deployment-airflow-scheduler.yaml index ff278aa2..c8fccf11 100644 --- a/charts/shipyard/templates/deployment-airflow-scheduler.yaml +++ b/charts/shipyard/templates/deployment-airflow-scheduler.yaml @@ -61,9 +61,9 @@ spec: env: - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql_airflow_db.user }} - key: AIRFLOW_DATABASE_URI + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI # Set to -1 to stop scheduler from going into crash loops args: ["scheduler", "-n", "-1" ] volumeMounts: diff --git a/charts/shipyard/templates/deployment-airflow-web.yaml b/charts/shipyard/templates/deployment-airflow-web.yaml deleted file mode 100644 index 648f5564..00000000 --- a/charts/shipyard/templates/deployment-airflow-web.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.manifests.deployment_airflow_web }} -{{- $envAll := . }} -{{- $serviceAccountName := "airflow-web" }} -{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} -{{- $mounts_airflow_web := .Values.pod.mounts.airflow_web.airflow_web }} -{{- $mounts_airflow_web_init := .Values.pod.mounts.airflow_web.init_container }} ---- -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: airflow-web - annotations: - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} -spec: - replicas: {{ .Values.pod.replicas.airflow.web }} -{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} - template: - metadata: - labels: -{{ tuple $envAll "airflow" "web" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: - configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }} - spec: - serviceAccountName: {{ $serviceAccountName }} - nodeSelector: - {{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }} - restartPolicy: Always - terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} - initContainers: -{{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: airflow-web - image: {{ .Values.images.tags.airflow }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.airflow.web | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - env: - - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql_airflow_db.user }} - key: AIRFLOW_DATABASE_URI - ports: - - containerPort: {{ .Values.network.airflow.web.port }} - args: ["webserver"] - readinessProbe: - tcpSocket: - port: {{ .Values.network.airflow.web.port }} - volumeMounts: - - name: airflow-etc - mountPath: {{ .Values.conf.airflow_config_file.path }} - subPath: airflow.cfg - readOnly: true - - name: shipyard-etc - mountPath: /usr/local/airflow/plugins/shipyard.conf - subPath: shipyard.conf - readOnly: true - - name: airflow-logs - mountPath: {{ .Values.conf.airflow.core.base_log_folder }} -{{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }} - volumes: - - name: airflow-etc - configMap: - name: airflow-etc - defaultMode: 0444 - - name: shipyard-etc - configMap: - name: shipyard-etc - defaultMode: 0444 - - name: airflow-bin - configMap: - name: airflow-bin - defaultMode: 0555 - - name: airflow-logs - emptyDir: {} -{{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }} -{{- end }} diff --git a/charts/shipyard/templates/deployment-shipyard.yaml b/charts/shipyard/templates/deployment-shipyard.yaml index 2d2700c3..a8cb104c 100644 --- a/charts/shipyard/templates/deployment-shipyard.yaml +++ b/charts/shipyard/templates/deployment-shipyard.yaml @@ -15,9 +15,10 @@ {{- if .Values.manifests.deployment_shipyard }} {{- $envAll := . }} +{{- $serviceAccountName := "shipyard" }} {{- $mounts_shipyard := .Values.pod.mounts.shipyard.shipyard }} {{- $mounts_shipyard_init := .Values.pod.mounts.shipyard.init_container }} -{{- $serviceAccountName := "shipyard" }} + {{ tuple $envAll "shipyard" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: apps/v1beta1 @@ -34,8 +35,10 @@ spec: labels: {{ tuple $envAll "shipyard" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} annotations: - configmap-bin-hash: {{ tuple "configmap-shipyard-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-shipyard-etc.yaml" . | include "helm-toolkit.utils.hash" }} + shipyard-configmap-bin-hash: {{ tuple "configmap-shipyard-bin.yaml" . | include "helm-toolkit.utils.hash" }} + shipyard-configmap-etc-hash: {{ tuple "configmap-shipyard-etc.yaml" . | include "helm-toolkit.utils.hash" }} + airflow-configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }} + airflow-configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }} spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: @@ -88,6 +91,28 @@ spec: - name: tmp-profiles mountPath: /tmp/profiles {{ end }} + - name: airflow-web + image: {{ .Values.images.tags.airflow }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.airflow.web | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI + args: ["webserver"] + volumeMounts: + - name: airflow-etc + mountPath: {{ .Values.conf.airflow_config_file.path }} + subPath: airflow.cfg + readOnly: true + - name: shipyard-etc + mountPath: /usr/local/airflow/plugins/shipyard.conf + subPath: shipyard.conf + readOnly: true + - name: airflow-logs + mountPath: {{ .Values.conf.airflow.core.base_log_folder }} {{ if $mounts_shipyard.volumeMounts }}{{ toYaml $mounts_shipyard.volumeMounts | indent 12 }}{{ end }} volumes: {{ if .Values.conf.shipyard.base.profiler }} @@ -100,5 +125,15 @@ spec: configMap: name: shipyard-etc defaultMode: 0444 + - name: airflow-etc + configMap: + name: airflow-etc + defaultMode: 0444 + - name: airflow-bin + configMap: + name: airflow-bin + defaultMode: 0555 + - name: airflow-logs + emptyDir: {} {{ if $mounts_shipyard.volumes }}{{ toYaml $mounts_shipyard.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/charts/shipyard/templates/ingress-airflow-api.yaml b/charts/shipyard/templates/ingress-airflow-api.yaml deleted file mode 100644 index f9c2deba..00000000 --- a/charts/shipyard/templates/ingress-airflow-api.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.ingress_airflow_api }} -{{- $envAll := . }} -{{- if .Values.network.airflow.ingress.public }} -{{- $backendServiceType := "airflow_web" }} -{{- $backendPort := "http" }} -{{- $ingressName := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -{{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -{{- $hostName := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -{{- $hostNameNamespaced := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }} -{{- $hostNameFull := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ $ingressName }} - annotations: - kubernetes.io/ingress.class: "nginx" - ingress.kubernetes.io/rewrite-target: / - nginx.ingress.kubernetes.io/proxy-read-timeout: {{ .Values.network.airflow.ingress.proxy_read_timeout | quote }} -spec: - rules: -{{- range $key1, $vHost := tuple $hostName $hostNameNamespaced $hostNameFull }} - - host: {{ $vHost }} - http: - paths: - - path: / - backend: - serviceName: {{ $backendName }} - servicePort: {{ $backendPort }} -{{- end }} -{{- end }} -{{- end }} diff --git a/charts/shipyard/templates/service-airflow-ingress.yaml b/charts/shipyard/templates/service-airflow-ingress.yaml deleted file mode 100644 index a3989067..00000000 --- a/charts/shipyard/templates/service-airflow-ingress.yaml +++ /dev/null @@ -1,33 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.service_airflow_ingress }} -{{- $envAll := . }} -{{- if .Values.network.airflow.ingress.public }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "airflow_web" "public" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -spec: - ports: - - name: http - port: 80 - selector: - app: ingress-api -{{- end }} -{{- end }} diff --git a/charts/shipyard/templates/service-airflow-web.yaml b/charts/shipyard/templates/service-airflow-web.yaml deleted file mode 100644 index bb68fe60..00000000 --- a/charts/shipyard/templates/service-airflow-web.yaml +++ /dev/null @@ -1,44 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. -Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.service_airflow_web }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "airflow_web" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} -spec: - ports: - {{ if .Values.network.airflow.web.enable_node_port }} - - name: http - nodePort: {{ .Values.network.airflow.web.node_port }} - port: {{ .Values.network.airflow.web.port }} - protocol: TCP - targetPort: {{ .Values.network.airflow.web.port }} - {{ else }} - - name: http - port: {{ .Values.network.airflow.web.port }} - protocol: TCP - targetPort: {{ .Values.network.airflow.web.port }} - {{ end }} - selector: -{{ tuple $envAll "airflow" "web" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - {{ if .Values.network.airflow.web.enable_node_port }} - type: NodePort - {{ end }} -{{- end }} diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml index c71b63eb..64a3997e 100644 --- a/charts/shipyard/templates/statefulset-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -101,9 +101,9 @@ spec: env: - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql_airflow_db.user }} - key: AIRFLOW_DATABASE_URI + secretKeyRef: + name: {{ .Values.secrets.postgresql_airflow_db.user }} + key: AIRFLOW_DATABASE_URI # Set to -1 to stop scheduler from going into crash loops args: ["scheduler", "-n", "-1" ] volumeMounts: diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 998a4295..5e780180 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -65,14 +65,6 @@ network: node_port: 31901 enable_node_port: false airflow: - ingress: - public: true - proxy_read_timeout: 600 - web: - name: airflow-web - port: 8080 - node_port: 32080 - enable_node_port: false worker: name: airflow-worker port: 8793 @@ -87,8 +79,6 @@ dependencies: services: - service: postgresql_shipyard_db endpoint: internal - - service: airflow_web - endpoint: internal shipyard_db_sync: jobs: - shipyard-db-init @@ -131,8 +121,6 @@ dependencies: - shipyard-ks-user - shipyard-ks-endpoints services: - - service: airflow_web - endpoint: internal - service: identity endpoint: internal - service: postgresql_shipyard_db @@ -213,21 +201,6 @@ endpoints: # tls: # crt: null # key: null - airflow_web: - name: airflow-web - hosts: - default: airflow-web-int - public: airflow-web - port: - airflow_web: - default: 8080 - public: 80 - path: - default: / - scheme: - default: http - host_fqdn_override: - default: null airflow_worker: name: airflow-worker hosts: @@ -394,7 +367,7 @@ conf: paste.filter_factory: keystonemiddleware.auth_token:filter_factory shipyard: base: - web_server: + web_server: http://localhost:8080/ pool_size: 15 pool_pre_ping: true pool_timeout: 30 @@ -507,7 +480,9 @@ conf: worker_precheck: "False" cli: api_client: airflow.api.client.local_client - # endpoint_url is extracted from endpoints by the configmap template + # if endpoint_url is not set, it is extracted from endpoints by the + # configmap template + endpoint_url: http://localhost/ api: auth_backend: airflow.api.auth.backend.default lineage: @@ -530,8 +505,12 @@ conf: # Shipyard is not using this default_hive_mapred_queue: "" webserver: - # base_url is extracted from endpoints by the configmap template - web_server_host: 0.0.0.0 + # if base_url is not set, is extracted from endpoints by the configmap + # template + base_url: http://localhost/ + # set web_server_host to 0.0.0.0 to bind to all interfaces. By default + # only bind to loopback + web_server_host: 127.0.0.1 web_server_port: 8080 web_server_ssl_cert: "" web_server_ssl_key: "" @@ -706,9 +685,6 @@ pod: init_container: null airflow_worker: airflow_scheduler: - airflow_web: - init_container: null - airflow_web: shipyard: init_container: null shipyard: @@ -722,7 +698,6 @@ pod: shipyard: api: 2 airflow: - web: 2 worker: 2 scheduler: 2 lifecycle: @@ -841,13 +816,6 @@ pod: requests: memory: "128Mi" cpu: "100m" - airflow: - limits: - memory: "128Mi" - cpu: "100m" - requests: - memory: "128Mi" - cpu: "100m" manifests: configmap_shipyard_bin: true @@ -858,9 +826,7 @@ manifests: # running the scheduler deployment_airflow_scheduler: true deployment_shipyard: true - deployment_airflow_web: true statefulset_airflow_worker: true - ingress_airflow_api: true ingress_shipyard_api: true job_shipyard_db_init: true job_shipyard_db_sync: true @@ -875,10 +841,8 @@ manifests: secret_ingress_tls: true secret_keystone: true secret_rabbitmq: true - service_airflow_ingress: true service_shipyard: true service_shipyard_ingress: true - service_airflow_web: true service_airflow_worker: true service_discovery_airflow_worker: true test_shipyard_api: true diff --git a/doc/source/_static/shipyard.policy.yaml.sample b/doc/source/_static/shipyard.policy.yaml.sample index 1692ac5b..63a4c9c7 100644 --- a/doc/source/_static/shipyard.policy.yaml.sample +++ b/doc/source/_static/shipyard.policy.yaml.sample @@ -40,10 +40,16 @@ # POST /api/v1.0/configdocs/{collection_id} #"workflow_orchestrator:create_configdocs": "rule:admin_required" -# Retrieve a collection of configuration documents +# Retrieve a collection of configuration documents with redacted +# secrets # GET /api/v1.0/configdocs/{collection_id} #"workflow_orchestrator:get_configdocs": "rule:admin_required" +# Retrieve a collection of configuration documents with cleartext +# secrets. +# GET /api/v1.0/configdocs/{collection_id} +#"workflow_orchestrator:get_configdocs_cleartext": "rule:admin_required" + # Move documents from the Shipyard buffer to the committed documents # POST /api/v1.0/commitconfigdocs #"workflow_orchestrator:commit_configdocs": "rule:admin_required" @@ -53,6 +59,11 @@ # GET /api/v1.0/renderedconfigdocs #"workflow_orchestrator:get_renderedconfigdocs": "rule:admin_required" +# Retrieve the configuration documents with cleartext secrets rendered +# by Deckhand into a complete design +# GET /api/v1.0/renderedconfigdocs +#"workflow_orchestrator:get_renderedconfigdocs_cleartext": "rule:admin_required" + # Retrieve the list of workflows (DAGs) that have been invoked in # Airflow, whether via Shipyard or scheduled # GET /api/v1.0/workflows diff --git a/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample b/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample index 1692ac5b..63a4c9c7 100644 --- a/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample +++ b/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample @@ -40,10 +40,16 @@ # POST /api/v1.0/configdocs/{collection_id} #"workflow_orchestrator:create_configdocs": "rule:admin_required" -# Retrieve a collection of configuration documents +# Retrieve a collection of configuration documents with redacted +# secrets # GET /api/v1.0/configdocs/{collection_id} #"workflow_orchestrator:get_configdocs": "rule:admin_required" +# Retrieve a collection of configuration documents with cleartext +# secrets. +# GET /api/v1.0/configdocs/{collection_id} +#"workflow_orchestrator:get_configdocs_cleartext": "rule:admin_required" + # Move documents from the Shipyard buffer to the committed documents # POST /api/v1.0/commitconfigdocs #"workflow_orchestrator:commit_configdocs": "rule:admin_required" @@ -53,6 +59,11 @@ # GET /api/v1.0/renderedconfigdocs #"workflow_orchestrator:get_renderedconfigdocs": "rule:admin_required" +# Retrieve the configuration documents with cleartext secrets rendered +# by Deckhand into a complete design +# GET /api/v1.0/renderedconfigdocs +#"workflow_orchestrator:get_renderedconfigdocs_cleartext": "rule:admin_required" + # Retrieve the list of workflows (DAGs) that have been invoked in # Airflow, whether via Shipyard or scheduled # GET /api/v1.0/workflows