Browse Source

Move Airflow web container into Shipyard pod

Moves the airflow web server container from its own pod into the
Shipyard pod. This removes exposed network surface area from the
Shipyard suite of software. Shipyard, after this change accesses the
Airflow API using localhost in the same k8s pod.

Change-Id: Ied4bd415a8d78c393b7256ead27a6a2176f4a2d6
changes/90/631890/9
Bryan Strassner 2 years ago
parent
commit
a11e962eef
10 changed files with 78 additions and 275 deletions
  1. +3
    -3
      charts/shipyard/templates/deployment-airflow-scheduler.yaml
  2. +0
    -92
      charts/shipyard/templates/deployment-airflow-web.yaml
  3. +38
    -3
      charts/shipyard/templates/deployment-shipyard.yaml
  4. +0
    -49
      charts/shipyard/templates/ingress-airflow-api.yaml
  5. +0
    -33
      charts/shipyard/templates/service-airflow-ingress.yaml
  6. +0
    -44
      charts/shipyard/templates/service-airflow-web.yaml
  7. +3
    -3
      charts/shipyard/templates/statefulset-airflow-worker.yaml
  8. +10
    -46
      charts/shipyard/values.yaml
  9. +12
    -1
      doc/source/_static/shipyard.policy.yaml.sample
  10. +12
    -1
      src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample

+ 3
- 3
charts/shipyard/templates/deployment-airflow-scheduler.yaml View File

@ -61,9 +61,9 @@ spec:
env:
- name: AIRFLOW_CONN_AIRFLOWS_OWN_DB
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
# Set to -1 to stop scheduler from going into crash loops
args: ["scheduler", "-n", "-1" ]
volumeMounts:


+ 0
- 92
charts/shipyard/templates/deployment-airflow-web.yaml View File

@ -1,92 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests.deployment_airflow_web }}
{{- $envAll := . }}
{{- $serviceAccountName := "airflow-web" }}
{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
{{- $mounts_airflow_web := .Values.pod.mounts.airflow_web.airflow_web }}
{{- $mounts_airflow_web_init := .Values.pod.mounts.airflow_web.init_container }}
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: airflow-web
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
replicas: {{ .Values.pod.replicas.airflow.web }}
{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
template:
metadata:
labels:
{{ tuple $envAll "airflow" "web" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }}
spec:
serviceAccountName: {{ $serviceAccountName }}
nodeSelector:
{{ .Values.labels.airflow.node_selector_key }}: {{ .Values.labels.airflow.node_selector_value }}
restartPolicy: Always
terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }}
initContainers:
{{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: airflow-web
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.airflow.web | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: AIRFLOW_CONN_AIRFLOWS_OWN_DB
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
ports:
- containerPort: {{ .Values.network.airflow.web.port }}
args: ["webserver"]
readinessProbe:
tcpSocket:
port: {{ .Values.network.airflow.web.port }}
volumeMounts:
- name: airflow-etc
mountPath: {{ .Values.conf.airflow_config_file.path }}
subPath: airflow.cfg
readOnly: true
- name: shipyard-etc
mountPath: /usr/local/airflow/plugins/shipyard.conf
subPath: shipyard.conf
readOnly: true
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{ if $mounts_airflow_web.volumeMounts }}{{ toYaml $mounts_airflow_web.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: airflow-etc
configMap:
name: airflow-etc
defaultMode: 0444
- name: shipyard-etc
configMap:
name: shipyard-etc
defaultMode: 0444
- name: airflow-bin
configMap:
name: airflow-bin
defaultMode: 0555
- name: airflow-logs
emptyDir: {}
{{ if $mounts_airflow_web.volumes }}{{ toYaml $mounts_airflow_web.volumes | indent 8 }}{{ end }}
{{- end }}

+ 38
- 3
charts/shipyard/templates/deployment-shipyard.yaml View File

@ -15,9 +15,10 @@
{{- if .Values.manifests.deployment_shipyard }}
{{- $envAll := . }}
{{- $serviceAccountName := "shipyard" }}
{{- $mounts_shipyard := .Values.pod.mounts.shipyard.shipyard }}
{{- $mounts_shipyard_init := .Values.pod.mounts.shipyard.init_container }}
{{- $serviceAccountName := "shipyard" }}
{{ tuple $envAll "shipyard" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: apps/v1beta1
@ -34,8 +35,10 @@ spec:
labels:
{{ tuple $envAll "shipyard" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
configmap-bin-hash: {{ tuple "configmap-shipyard-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-shipyard-etc.yaml" . | include "helm-toolkit.utils.hash" }}
shipyard-configmap-bin-hash: {{ tuple "configmap-shipyard-bin.yaml" . | include "helm-toolkit.utils.hash" }}
shipyard-configmap-etc-hash: {{ tuple "configmap-shipyard-etc.yaml" . | include "helm-toolkit.utils.hash" }}
airflow-configmap-bin-hash: {{ tuple "configmap-airflow-bin.yaml" . | include "helm-toolkit.utils.hash" }}
airflow-configmap-etc-hash: {{ tuple "configmap-airflow-etc.yaml" . | include "helm-toolkit.utils.hash" }}
spec:
serviceAccountName: {{ $serviceAccountName }}
nodeSelector:
@ -88,6 +91,28 @@ spec:
- name: tmp-profiles
mountPath: /tmp/profiles
{{ end }}
- name: airflow-web
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.airflow.web | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: AIRFLOW_CONN_AIRFLOWS_OWN_DB
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
args: ["webserver"]
volumeMounts:
- name: airflow-etc
mountPath: {{ .Values.conf.airflow_config_file.path }}
subPath: airflow.cfg
readOnly: true
- name: shipyard-etc
mountPath: /usr/local/airflow/plugins/shipyard.conf
subPath: shipyard.conf
readOnly: true
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{ if $mounts_shipyard.volumeMounts }}{{ toYaml $mounts_shipyard.volumeMounts | indent 12 }}{{ end }}
volumes:
{{ if .Values.conf.shipyard.base.profiler }}
@ -100,5 +125,15 @@ spec:
configMap:
name: shipyard-etc
defaultMode: 0444
- name: airflow-etc
configMap:
name: airflow-etc
defaultMode: 0444
- name: airflow-bin
configMap:
name: airflow-bin
defaultMode: 0555
- name: airflow-logs
emptyDir: {}
{{ if $mounts_shipyard.volumes }}{{ toYaml $mounts_shipyard.volumes | indent 8 }}{{ end }}
{{- end }}

+ 0
- 49
charts/shipyard/templates/ingress-airflow-api.yaml View File

@ -1,49 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.ingress_airflow_api }}
{{- $envAll := . }}
{{- if .Values.network.airflow.ingress.public }}
{{- $backendServiceType := "airflow_web" }}
{{- $backendPort := "http" }}
{{- $ingressName := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
{{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
{{- $hostName := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
{{- $hostNameNamespaced := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
{{- $hostNameFull := tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: {{ $ingressName }}
annotations:
kubernetes.io/ingress.class: "nginx"
ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/proxy-read-timeout: {{ .Values.network.airflow.ingress.proxy_read_timeout | quote }}
spec:
rules:
{{- range $key1, $vHost := tuple $hostName $hostNameNamespaced $hostNameFull }}
- host: {{ $vHost }}
http:
paths:
- path: /
backend:
serviceName: {{ $backendName }}
servicePort: {{ $backendPort }}
{{- end }}
{{- end }}
{{- end }}

+ 0
- 33
charts/shipyard/templates/service-airflow-ingress.yaml View File

@ -1,33 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.service_airflow_ingress }}
{{- $envAll := . }}
{{- if .Values.network.airflow.ingress.public }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ tuple "airflow_web" "public" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
spec:
ports:
- name: http
port: 80
selector:
app: ingress-api
{{- end }}
{{- end }}

+ 0
- 44
charts/shipyard/templates/service-airflow-web.yaml View File

@ -1,44 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.service_airflow_web }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ tuple "airflow_web" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
spec:
ports:
{{ if .Values.network.airflow.web.enable_node_port }}
- name: http
nodePort: {{ .Values.network.airflow.web.node_port }}
port: {{ .Values.network.airflow.web.port }}
protocol: TCP
targetPort: {{ .Values.network.airflow.web.port }}
{{ else }}
- name: http
port: {{ .Values.network.airflow.web.port }}
protocol: TCP
targetPort: {{ .Values.network.airflow.web.port }}
{{ end }}
selector:
{{ tuple $envAll "airflow" "web" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{ if .Values.network.airflow.web.enable_node_port }}
type: NodePort
{{ end }}
{{- end }}

+ 3
- 3
charts/shipyard/templates/statefulset-airflow-worker.yaml View File

@ -101,9 +101,9 @@ spec:
env:
- name: AIRFLOW_CONN_AIRFLOWS_OWN_DB
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
# Set to -1 to stop scheduler from going into crash loops
args: ["scheduler", "-n", "-1" ]
volumeMounts:


+ 10
- 46
charts/shipyard/values.yaml View File

@ -65,14 +65,6 @@ network:
node_port: 31901
enable_node_port: false
airflow:
ingress:
public: true
proxy_read_timeout: 600
web:
name: airflow-web
port: 8080
node_port: 32080
enable_node_port: false
worker:
name: airflow-worker
port: 8793
@ -87,8 +79,6 @@ dependencies:
services:
- service: postgresql_shipyard_db
endpoint: internal
- service: airflow_web
endpoint: internal
shipyard_db_sync:
jobs:
- shipyard-db-init
@ -131,8 +121,6 @@ dependencies:
- shipyard-ks-user
- shipyard-ks-endpoints
services:
- service: airflow_web
endpoint: internal
- service: identity
endpoint: internal
- service: postgresql_shipyard_db
@ -213,21 +201,6 @@ endpoints:
# tls:
# crt: null
# key: null
airflow_web:
name: airflow-web
hosts:
default: airflow-web-int
public: airflow-web
port:
airflow_web:
default: 8080
public: 80
path:
default: /
scheme:
default: http
host_fqdn_override:
default: null
airflow_worker:
name: airflow-worker
hosts:
@ -394,7 +367,7 @@ conf:
paste.filter_factory: keystonemiddleware.auth_token:filter_factory
shipyard:
base:
web_server:
web_server: http://localhost:8080/
pool_size: 15
pool_pre_ping: true
pool_timeout: 30
@ -507,7 +480,9 @@ conf:
worker_precheck: "False"
cli:
api_client: airflow.api.client.local_client
# endpoint_url is extracted from endpoints by the configmap template
# if endpoint_url is not set, it is extracted from endpoints by the
# configmap template
endpoint_url: http://localhost/
api:
auth_backend: airflow.api.auth.backend.default
lineage:
@ -530,8 +505,12 @@ conf:
# Shipyard is not using this
default_hive_mapred_queue: ""
webserver:
# base_url is extracted from endpoints by the configmap template
web_server_host: 0.0.0.0
# if base_url is not set, is extracted from endpoints by the configmap
# template
base_url: http://localhost/
# set web_server_host to 0.0.0.0 to bind to all interfaces. By default
# only bind to loopback
web_server_host: 127.0.0.1
web_server_port: 8080
web_server_ssl_cert: ""
web_server_ssl_key: ""
@ -706,9 +685,6 @@ pod:
init_container: null
airflow_worker:
airflow_scheduler:
airflow_web:
init_container: null
airflow_web:
shipyard:
init_container: null
shipyard:
@ -722,7 +698,6 @@ pod:
shipyard:
api: 2
airflow:
web: 2
worker: 2
scheduler: 2
lifecycle:
@ -841,13 +816,6 @@ pod:
requests:
memory: "128Mi"
cpu: "100m"
airflow:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
manifests:
configmap_shipyard_bin: true
@ -858,9 +826,7 @@ manifests:
# running the scheduler
deployment_airflow_scheduler: true
deployment_shipyard: true
deployment_airflow_web: true
statefulset_airflow_worker: true
ingress_airflow_api: true
ingress_shipyard_api: true
job_shipyard_db_init: true
job_shipyard_db_sync: true
@ -875,10 +841,8 @@ manifests:
secret_ingress_tls: true
secret_keystone: true
secret_rabbitmq: true
service_airflow_ingress: true
service_shipyard: true
service_shipyard_ingress: true
service_airflow_web: true
service_airflow_worker: true
service_discovery_airflow_worker: true
test_shipyard_api: true

+ 12
- 1
doc/source/_static/shipyard.policy.yaml.sample View File

@ -40,10 +40,16 @@
# POST /api/v1.0/configdocs/{collection_id}
#"workflow_orchestrator:create_configdocs": "rule:admin_required"
# Retrieve a collection of configuration documents
# Retrieve a collection of configuration documents with redacted
# secrets
# GET /api/v1.0/configdocs/{collection_id}
#"workflow_orchestrator:get_configdocs": "rule:admin_required"
# Retrieve a collection of configuration documents with cleartext
# secrets.
# GET /api/v1.0/configdocs/{collection_id}
#"workflow_orchestrator:get_configdocs_cleartext": "rule:admin_required"
# Move documents from the Shipyard buffer to the committed documents
# POST /api/v1.0/commitconfigdocs
#"workflow_orchestrator:commit_configdocs": "rule:admin_required"
@ -53,6 +59,11 @@
# GET /api/v1.0/renderedconfigdocs
#"workflow_orchestrator:get_renderedconfigdocs": "rule:admin_required"
# Retrieve the configuration documents with cleartext secrets rendered
# by Deckhand into a complete design
# GET /api/v1.0/renderedconfigdocs
#"workflow_orchestrator:get_renderedconfigdocs_cleartext": "rule:admin_required"
# Retrieve the list of workflows (DAGs) that have been invoked in
# Airflow, whether via Shipyard or scheduled
# GET /api/v1.0/workflows


+ 12
- 1
src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample View File

@ -40,10 +40,16 @@
# POST /api/v1.0/configdocs/{collection_id}
#"workflow_orchestrator:create_configdocs": "rule:admin_required"
# Retrieve a collection of configuration documents
# Retrieve a collection of configuration documents with redacted
# secrets
# GET /api/v1.0/configdocs/{collection_id}
#"workflow_orchestrator:get_configdocs": "rule:admin_required"
# Retrieve a collection of configuration documents with cleartext
# secrets.
# GET /api/v1.0/configdocs/{collection_id}
#"workflow_orchestrator:get_configdocs_cleartext": "rule:admin_required"
# Move documents from the Shipyard buffer to the committed documents
# POST /api/v1.0/commitconfigdocs
#"workflow_orchestrator:commit_configdocs": "rule:admin_required"
@ -53,6 +59,11 @@
# GET /api/v1.0/renderedconfigdocs
#"workflow_orchestrator:get_renderedconfigdocs": "rule:admin_required"
# Retrieve the configuration documents with cleartext secrets rendered
# by Deckhand into a complete design
# GET /api/v1.0/renderedconfigdocs
#"workflow_orchestrator:get_renderedconfigdocs_cleartext": "rule:admin_required"
# Retrieve the list of workflows (DAGs) that have been invoked in
# Airflow, whether via Shipyard or scheduled
# GET /api/v1.0/workflows


Loading…
Cancel
Save