A cluster lifecycle orchestrator for Airship.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

860 lines
23 KiB

# Copyright 2017 The Openstack-Helm Authors.
# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file provides defaults for shipyard and airflow
# Indicate whether it is production or development environment
prod_environment: true
labels:
job:
node_selector_key: ucp-control-plane
node_selector_value: enabled
shipyard:
node_selector_key: ucp-control-plane
node_selector_value: enabled
airflow:
node_selector_key: ucp-control-plane
node_selector_value: enabled
images:
tags:
airflow: quay.io/airshipit/airflow:latest
shipyard: quay.io/airshipit/shipyard:latest
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
shipyard_db_init: docker.io/postgres:9.5
shipyard_db_sync: quay.io/airshipit/shipyard:latest
airflow_db_init: docker.io/postgres:9.5
airflow_db_sync: quay.io/airshipit/airflow:latest
ks_user: docker.io/openstackhelm/heat:ocata
ks_service: docker.io/openstackhelm/heat:ocata
ks_endpoints: docker.io/openstackhelm/heat:ocata
image_repo_sync: docker.io/docker:17.07.0
pull_policy: "IfNotPresent"
local_registry:
active: false
exclude:
- dep_check
- image_repo_sync
release_group: null
network:
shipyard:
ingress:
public: true
classes:
namespace: "nginx"
cluster: "nginx-cluster"
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
nginx.ingress.kubernetes.io/proxy-body-size: "10m"
node_port: 31901
enable_node_port: false
airflow:
ingress:
public: true
proxy_read_timeout: 600
flower:
name: airflow-flower
port: 5555
enable_node_port: false
web:
name: airflow-web
port: 8080
node_port: 32080
enable_node_port: false
worker:
name: airflow-worker
port: 8793
enable_node_port: false
dependencies:
static:
shipyard_db_init:
jobs:
- airflow-db-init
- airflow-db-sync
services:
- service: postgresql_shipyard_db
endpoint: internal
- service: airflow_flower
endpoint: internal
- service: airflow_web
endpoint: internal
shipyard_db_sync:
jobs:
- shipyard-db-init
services:
- service: postgresql_shipyard_db
endpoint: internal
airflow_db_init:
services:
- service: postgresql_airflow_db
endpoint: internal
airflow_db_sync:
jobs:
- airflow-db-init
services:
- service: postgresql_airflow_db
endpoint: internal
ks_user:
services:
- service: identity
endpoint: internal
ks_service:
services:
- service: identity
endpoint: internal
ks_endpoints:
jobs:
- shipyard-ks-service
services:
- service: identity
endpoint: internal
shipyard:
jobs:
- shipyard-db-init
- shipyard-db-sync
- shipyard-ks-endpoints
- shipyard-ks-user
- shipyard-ks-endpoints
services:
- service: airflow_flower
endpoint: internal
- service: airflow_web
endpoint: internal
- service: identity
endpoint: internal
- service: postgresql_shipyard_db
endpoint: internal
airflow_server:
jobs:
- airflow-db-init
- airflow-db-sync
services:
- service: postgresql_airflow_db
endpoint: internal
- service: oslo_messaging
endpoint: internal
volume_worker:
class_name: general
size: 5Gi
logrotate:
days_before_deletion: 30
# typically overriden by environmental
# values, but should include all endpoints
# required by this chart
endpoints:
cluster_domain_suffix: cluster.local
identity:
name: keystone
auth:
shipyard:
region_name: RegionOne
role: admin
project_name: service
project_domain_name: default
user_domain_name: default
username: shipyard
password: password
admin:
region_name: RegionOne
project_name: admin
password: password
username: admin
user_domain_name: default
project_domain_name: default
hosts:
default: keystone
internal: keystone-api
path:
default: /v3
scheme:
default: http
port:
api:
default: 80
internal: 5000
host_fqdn_override:
default: null
shipyard:
name: shipyard
hosts:
default: shipyard-int
public: shipyard-api
port:
api:
default: 9000
public: 80
path:
default: /api/v1.0
scheme:
default: http
host_fqdn_override:
default: null
# NOTE(bryan-strassner): this chart supports TLS for fqdn over-ridden public
# endpoints using the following format:
# public:
# host: null
# tls:
# crt: null
# key: null
airflow_web:
name: airflow-web
hosts:
default: airflow-web-int
public: airflow-web
port:
airflow_web:
default: 8080
public: 80
path:
default: /
scheme:
default: http
host_fqdn_override:
default: null
airflow_flower:
name: airflow-flower
hosts:
default: airflow-flower
port:
airflow_flower:
default: 5555
path:
default: /
scheme:
default: http
host_fqdn_override:
default: null
airflow_worker:
name: airflow-worker
hosts:
default: airflow-worker
discovery: airflow-worker-discovery
host_fqdn_override:
default: null
path: null
scheme: 'http'
port:
airflow_worker:
default: 8793
postgresql_shipyard_db:
name: postgresql_shipyard_db
auth:
admin:
username: postgres
password: password
user:
username: shipyard
password: password
database: shipyard
hosts:
default: postgresql
path: /shipyard
scheme: postgresql+psycopg2
port:
postgresql:
default: 5432
host_fqdn_override:
default: null
postgresql_airflow_db:
name: postgresql_airflow_db
auth:
admin:
username: postgres
password: password
user:
username: airflow
password: password
database: airflow
hosts:
default: postgresql
path: /airflow
scheme: postgresql+psycopg2
port:
postgresql:
default: 5432
host_fqdn_override:
default: null
postgresql_airflow_celery_db:
name: postgresql_airflow_celery_db
auth:
admin:
username: postgres
password: password
user:
username: airflow
password: password
database: airflow
hosts:
default: postgresql
path: /airflow
scheme: db+postgresql
port:
postgresql:
default: 5432
host_fqdn_override:
default: null
oslo_messaging:
auth:
user:
username: rabbitmq
password: password
hosts:
default: rabbitmq
host_fqdn_override:
default: null
path: /
scheme: amqp
port:
amqp:
default: 5672
oslo_cache:
hosts:
default: memcached
host_fqdn_override:
default: null
port:
memcache:
default: 11211
secrets:
identity:
admin: shipyard-keystone-admin
shipyard: shipyard-keystone-user
postgresql_shipyard_db:
admin: shipyard-db-admin
user: shipyard-db-user
postgresql_airflow_db:
admin: airflow-db-admin
user: airflow-db-user
tls:
shipyard:
shipyard:
public: shipyard-tls-public
conf:
uwsgi:
threads: 1
workers: 4
policy:
admin_create: role:admin or role:admin_ucp
admin_read_access: rule:admin_create or role:admin_ucp_viewer
workflow_orchestrator:list_actions: rule:admin_read_access
workflow_orchestrator:create_action: rule:admin_create
workflow_orchestrator:get_action: rule:admin_read_access
workflow_orchestrator:get_action_step: rule:admin_read_access
workflow_orchestrator:get_action_step_logs: rule:admin_read_access
workflow_orchestrator:get_action_validation: rule:admin_read_access
workflow_orchestrator:invoke_action_control: rule:admin_create
workflow_orchestrator:get_configdocs_status: rule:admin_read_access
workflow_orchestrator:create_configdocs: rule:admin_create
workflow_orchestrator:get_configdocs: rule:admin_read_access
workflow_orchestrator:commit_configdocs: rule:admin_create
workflow_orchestrator:get_renderedconfigdocs: rule:admin_read_access
workflow_orchestrator:list_workflows: rule:admin_read_access
workflow_orchestrator:get_workflow: rule:admin_read_access
workflow_orchestrator:get_notedetails: rule:admin_read_access
workflow_orchestrator:get_site_statuses: rule:admin_read_access
workflow_orchestrator:action_deploy_site: rule:admin_create
workflow_orchestrator:action_update_site: rule:admin_create
workflow_orchestrator:action_update_software: rule:admin_create
workflow_orchestrator:action_redeploy_server: rule:admin_create
workflow_orchestrator:action_relabel_nodes: rule:admin_create
paste:
app:shipyard-api:
paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard
pipeline:main:
pipeline: authtoken shipyard-api
filter:authtoken:
paste.filter_factory: keystonemiddleware.auth_token:filter_factory
shipyard:
base:
web_server:
pool_size: 15
pool_pre_ping: true
pool_timeout: 30
pool_overflow: 10
connection_recycle: -1
profiler: false
shipyard:
service_type: shipyard
deckhand:
service_type: deckhand
armada:
service_type: armada
drydock:
service_type: physicalprovisioner
promenade:
service_type: kubernetesprovisioner
keystone_authtoken:
delay_auth_decision: true
auth_type: password
auth_section: keystone_authtoken
auth_version: v3
memcache_security_strategy: ENCRYPT
requests_config:
airflow_log_connect_timeout: 5
airflow_log_read_timeout: 300
deckhand_client_connect_timeout: 5
deckhand_client_read_timeout: 300
validation_connect_timeout: 5
validation_read_timeout: 300
notes_connect_timeout: 5
notes_read_timeout: 10
drydock_client_connect_timeout: 20
drydock_client_read_timeout: 300
airflow:
worker_endpoint_scheme: 'http'
worker_port: 8793
k8s_logs:
ucp_namespace: 'ucp'
oslo_policy:
policy_file: /etc/shipyard/policy.yaml
# If non-existent rule is used, the request should be denied. The
# deny_all rule is hard coded in the policy.py code to allow no access.
policy_default_rule: deny_all
airflow_config_file:
path: /usr/local/airflow/airflow.cfg
airflow:
# NOTE: Airflow 1.10 introduces a need to declare all config options:
# https://issues.apache.org/jira/browse/AIRFLOW-3099
core:
airflow_home: /usr/local/airflow
dags_folder: /usr/local/airflow/dags
base_log_folder: /usr/local/airflow/logs
remote_logging: "False"
remote_log_conn_id: ""
remote_base_log_folder: ""
encrypt_s3_logs: "False"
logging_level: "INFO"
fab_logging_level: "WARN"
# TODO(bryan-strassner) Use this for custom log formatting!
logging_config_class: ""
# NOTE: Airflow 1.10 introduces extra newline characters between log
# records. Version 1.10.1 should resolve this issue
# https://issues.apache.org/jira/browse/AIRFLOW-1917
#
# NOTE: The log format ends up repeated for each log record that we log
# in our custom operators, once for the logging_mixin class of
# Airflow itself, and once again for the message we want to log.
# E.g.:
# 2018-09-21 19:38:48,950 INFO logging_mixin(95) write - 2018-09-21 19:38:48,950 INFO deployment_configuration_operator(135) get_doc - Deckhand Client acquired
#
# NOTE: Updated from default to match Shipyard logging as much as
# possible without more aggressive techniques
#
log_format: "%%(asctime)s %%(levelname)-8s %%(module)s(%%(lineno)d) %%(funcName)s - %%(message)s"
simple_log_format: "%%(asctime)s %%(levelname)s - %%(message)s"
log_filename_template: "{{ ti.dag_id }}/{{ ti.task_id }}/{{ execution_date.strftime('%%Y-%%m-%%dT%%H:%%M:%%S') }}/{{ try_number }}.log"
log_processor_filename_template: "{{ filename }}.log"
hostname_callable: "socket:getfqdn"
default_timezone: "utc"
executor: "CeleryExecutor"
# sql_alchemy_conn is extracted from endpoints by the configmap template
sql_alchemy_pool_enabled: "True"
sql_alchemy_pool_size: 5
sql_alchemy_pool_recycle: 1800
sql_alchemy_reconnect_timeout: 30
parallelism: 32
dag_concurrency: 16
dags_are_paused_at_creation: "False"
non_pooled_task_slot_count: 128
max_active_runs_per_dag: 16
load_examples: "False"
plugins_folder: /usr/local/airflow/plugins
fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs=
donot_pickle: "False"
dagbag_import_timeout: 30
# NOTE: Versions after 1.10 will change this to StandardTaskRunner
task_runner: "BashTaskRunner"
default_impersonation: ""
security: ""
secure_mode: "True"
unit_test_mode: "False"
task_log_reader: "task"
enable_xcom_pickling: "False"
killed_task_cleanup_time: 60
dag_run_conf_overrides_params: "False"
cli:
api_client: airflow.api.client.local_client
# endpoint_url is extracted from endpoints by the configmap template
api:
auth_backend: airflow.api.auth.backend.default
lineage:
# Shipyard is not using this
backend: ""
atlas:
# Shipyard is not using this
sasl_enabled: "False"
host: ""
port: 21000
username: ""
password: ""
operators:
default_owner: "Airflow"
default_cpus: 1
default_ram: 512
default_disk: 512
default_gpus: 0
hive:
# Shipyard is not using this
default_hive_mapred_queue: ""
webserver:
# base_url is extracted from endpoints by the configmap template
web_server_host: 0.0.0.0
web_server_port: 8080
web_server_ssl_cert: ""
web_server_ssl_key: ""
web_server_master_timeout: 120
web_server_worker_timeout: 120
worker_refresh_batch_size: 1
worker_refresh_interval: 30
secret_key: "temporary_key"
workers: 4
worker_class: "sync"
access_logfile: "-"
error_logfile: "-"
expose_config: "False"
authenticate: "False"
filter_by_owner: "False"
owner_mode: "user"
dag_default_view: "tree"
dag_orientation: "LR"
demo_mode: "False"
log_fetch_timeout_sec: 10
hide_paused_dags_by_default: "False"
page_size: 100
rbac: "False"
navbar_color: "#007A87"
default_dag_run_display_number: 25
email:
# Shipyard is not using this
email_backend: airflow.utils.send_email_smtp
smtp:
# Shipyard is not using this
smtp_host: "localhost"
smtp_starttls: "True"
smtp_ssl: "False"
smtp_user: "airflow"
smtp_password: "airflow"
smtp_port: 25
smtp_mail_from: airflow@airflow.local
celery:
celery_app_name: airflow.executors.celery_executor
worker_concurrency: 16
worker_log_server_port: 8793
# broker_url is extracted from endpoints by the configmap template
# result_backend is extracted from endpoints by the configmap template
flower_host: 0.0.0.0
flower_url_prefix:
flower_port: 5555
default_queue: "default"
celery_config_options: airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
# TODO: Enable this for security
ssl_active: "False"
ssl_key: ""
ssl_cert: ""
ssl_cacert: ""
celery_broker_transport_options:
visibility_timeout: 21600
dask:
# Shipyard is not using this
cluster_adresss: "127.0.0.1:8786"
tls_ca: ""
tls_cert: ""
tls_key: ""
scheduler:
job_heartbeat_sec: 5
scheduler_heartbeat_sec: 5
run_duration: -1
# Check for pending tasks no more than every 5 seconds
min_file_process_interval: 5
# This is part of 1.10, but disabled in 1.10.1 (pending) See:
# https://github.com/apache/incubator-airflow/blob/master/UPDATING.md#min_file_parsing_loop_time-config-option-temporarily-disabled
min_file_parsing_loop_time: 1
dag_dir_list_interval: 300
# Stats for the scheduler are minimally useful - every 5 mins is enough
print_stats_interval: 300
child_process_log_directory: /usr/local/airflow/logs/scheduler
scheduler_zombie_task_threshold: 300
catchup_by_default: "True"
max_tis_per_query: 512
statsd_on: "False"
statsd_host: "localhost"
statsd_port: 8125
statsd_prefix: "airflow"
# Shipyard's use of Airflow is low volume. 1 Thread is probably enough.
max_threads: 1
authenticate: "False"
ldap:
# Shipyard is not using this
uri: ""
user_filter: "objectClass=*"
user_name_attr: "uid"
group_member_attr: "memberOf"
superuser_filter: ""
data_profiler_filter: ""
bind_user: "cn=Manager,dc=example,dc=com"
bind_password: "insecure"
basedn: "dc=example,dc=com"
cacert: "/etc/ca/ldap_ca.crt"
search_scope: "LEVEL"
mesos:
# Shipyard is not using this
master: ""
framework_name: ""
task_cpu: ""
task_memory: ""
checkpoint: ""
authenticate: ""
kerberos:
# Shipyard is not using this
ccache: ""
principal: ""
reinit_frequency: ""
kinit_path: ""
keytab: ""
github_enterprise:
# Shipyard is not using this
api_rev: v3
admin:
hide_sensitive_variable_fields: "True"
elasticsearch:
# Shipyard is not using this
elasticsearch_host: ""
elasticsearch_log_id_template: ""
elasticsearch_end_of_log_mark: ""
kubernetes:
# Shipyard is not using this (maybe future for spawning own workers)
worker_container_repository: ""
worker_container_tag: ""
delete_worker_pods: "True"
namespace: "default"
airflow_configmap: ""
dags_volume_subpath: ""
dags_volume_claim: ""
logs_volume_subpath: ""
logs_volume_claim: ""
git_repo: ""
git_branch: ""
git_user: ""
git_password: ""
git_subpath: ""
git_sync_container_repository: ""
git_sync_container_tag: ""
git_sync_init_container_name: ""
worker_service_account_name: ""
image_pull_secrets: ""
gcp_service_account_keys: ""
in_cluster: ""
kubernetes_secrets:
#Shipyard is not using this
# End of Airflow config options
pod:
mounts:
dag_path: /home/ubuntu/workbench/dags
plugin_path: /home/ubuntu/workbench/plugins
log_path: /home/ubuntu/workbench/logs
airflow_flower:
init_container: null
airflow_flower:
airflow_scheduler:
init_container: null
airflow_scheduler:
airflow_worker:
init_container: null
airflow_worker:
airflow_web:
init_container: null
airflow_web:
shipyard:
init_container: null
shipyard:
shipyard_db_init:
init_container: null
shipyard_db_init:
shipyard_db_sync:
init_container: null
shipyard_db_sync:
replicas:
shipyard:
api: 2
airflow:
web: 2
worker: 2
flower: 2
scheduler: 2
lifecycle:
upgrades:
deployments:
revision_history: 3
pod_replacement_strategy: RollingUpdate
rolling_update:
max_unavailable: 1
max_surge: 3
termination_grace_period:
airflow:
timeout: 30
shipyard:
timeout: 30
resources:
enabled: false
airflow:
flower:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
logrotate:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
scheduler:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
web:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
worker:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
shipyard_api:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
jobs:
airflow_db_init:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
airflow_db_sync:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
ks_endpoints:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
ks_service:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
ks_user:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
shipyard_db_init:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
shipyard_db_sync:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
test:
shipyard:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
airflow:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
manifests:
configmap_shipyard_bin: true
configmap_shipyard_etc: true
configmap_airflow_bin: true
configmap_airflow_etc: true
deployment_airflow_flower: true
deployment_airflow_scheduler: true
deployment_shipyard: true
deployment_airflow_web: true
statefulset_airflow_worker: true
ingress_airflow_api: true
ingress_shipyard_api: true
job_shipyard_db_init: true
job_shipyard_db_sync: true
job_airflow_db_init: true
job_airflow_db_sync: true
job_ks_endpoints: true
job_ks_service: true
job_ks_user: true
secret_airflow_db: true
secret_shipyard_db: true
secret_ingress_tls: true
secret_keystone: true
service_airflow_ingress: true
service_airflow_flower: true
service_shipyard: true
service_shipyard_ingress: true
service_airflow_web: true
service_airflow_worker: true
service_discovery_airflow_worker: true
test_shipyard_api: true