diff --git a/Makefile b/Makefile index 44cff068..d923c196 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local USE_PROXY ?= false AIRFLOW_SRC ?= +AIRFLOW_HOME ?= DISTRO_BASE_IMAGE ?= DISTRO ?= ubuntu_xenial @@ -86,6 +87,7 @@ run: _BASE_IMAGE_ARG := $(if $(DISTRO_BASE_IMAGE),--build-arg FROM="${DISTRO_BASE_IMAGE}" ,) ifeq ($(IMAGE_NAME), airflow) _AIRFLOW_SRC_ARG := $(if $(AIRFLOW_SRC),--build-arg AIRFLOW_SRC="${AIRFLOW_SRC}" ,) + _AIRFLOW_HOME_ARG := $(if $(AIRFLOW_HOME),--build-arg AIRFLOW_HOME="${AIRFLOW_HOME}" ,) endif .PHONY: build @@ -98,6 +100,7 @@ ifeq ($(USE_PROXY), true) -f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \ $(_BASE_IMAGE_ARG) \ $(_AIRFLOW_SRC_ARG) \ + $(_AIRFLOW_HOME_ARG) \ --build-arg http_proxy=$(PROXY) \ --build-arg https_proxy=$(PROXY) \ --build-arg HTTP_PROXY=$(PROXY) \ @@ -113,6 +116,7 @@ else -f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \ $(_BASE_IMAGE_ARG) \ $(_AIRFLOW_SRC_ARG) \ + $(_AIRFLOW_HOME_ARG) \ --build-arg ctx_base=$(BUILD_CTX) . endif ifeq ($(PUSH_IMAGE), true) diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 6cba3e0d..08ba0dcb 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -454,7 +454,9 @@ conf: # NOTE: Airflow 1.10 introduces a need to declare all config options: # https://issues.apache.org/jira/browse/AIRFLOW-3099 core: - airflow_home: /usr/local/airflow + # core.airflow_home is not used in 1.10.3 and later. + # Envrionment variable AIRFLOW_HOME is used instead. + #airflow_home: /usr/local/airflow dags_folder: /usr/local/airflow/dags base_log_folder: /usr/local/airflow/logs remote_logging: "False" @@ -497,15 +499,17 @@ conf: parallelism: 32 dag_concurrency: 8 dags_are_paused_at_creation: "False" - non_pooled_task_slot_count: 128 + # In 1.10.4, non_pooled_task_slot_count and non_pooled_backfill_task_slot_count are + # removed in favor of default_pool, which is initialized with 128 slots by default. + #non_pooled_task_slot_count: 128 max_active_runs_per_dag: 8 load_examples: "False" plugins_folder: /usr/local/airflow/plugins fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs= donot_pickle: "False" dagbag_import_timeout: 30 - task_runner: "BashTaskRunner" - # task_runner: "StandardTaskRunner" -- coming soon? + # BashTaskRunner has been renamed to StandardTaskRunner from 1.10.3 + task_runner: "StandardTaskRunner" default_impersonation: "" security: "" secure_mode: "True" @@ -683,9 +687,9 @@ conf: hide_sensitive_variable_fields: "True" elasticsearch: # Shipyard is not using this - elasticsearch_host: "" - elasticsearch_log_id_template: "" - elasticsearch_end_of_log_mark: "" + host: "" + log_id_template: "" + end_of_log_mark: "" kubernetes: # Shipyard is not using this (maybe future for spawning own workers) worker_container_repository: "" diff --git a/images/airflow/Dockerfile.opensuse_15 b/images/airflow/Dockerfile.opensuse_15 index 6931f907..0e8772eb 100644 --- a/images/airflow/Dockerfile.opensuse_15 +++ b/images/airflow/Dockerfile.opensuse_15 @@ -40,11 +40,14 @@ EXPOSE $WORKER_PORT # Set ARG for usage during build ARG AIRFLOW_HOME=/usr/local/airflow -ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1" +ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4" ARG ctx_base=src/bin # Kubectl version -ARG KUBECTL_VERSION=1.10.2 +ARG KUBECTL_VERSION=1.16.2 + +# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated +ENV AIRFLOW_HOME=${AIRFLOW_HOME} RUN set -ex && \ zypper -q update -y ;\ diff --git a/images/airflow/Dockerfile.ubuntu_xenial b/images/airflow/Dockerfile.ubuntu_xenial index 5ef8d435..e3d965dd 100644 --- a/images/airflow/Dockerfile.ubuntu_xenial +++ b/images/airflow/Dockerfile.ubuntu_xenial @@ -40,12 +40,15 @@ EXPOSE $WORKER_PORT # Set ARG for usage during build ARG AIRFLOW_HOME=/usr/local/airflow -ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1" +ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4" ARG DEBIAN_FRONTEND=noninteractive ARG ctx_base=src/bin # Kubectl version -ARG KUBECTL_VERSION=1.10.2 +ARG KUBECTL_VERSION=1.16.2 + +# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated +ENV AIRFLOW_HOME=${AIRFLOW_HOME} RUN set -ex && \ apt-get -qq update && \