Browse Source

(airflow) - Apache airflow uplift to 1.10.4

Updated airflow config and Dockerfiles for apache airflow 1.10.4
For details see: https://github.com/apache/airflow/blob/master/UPDATING.md

Also updated the kubectl to 1.16.2, as part of kubernetes 1.16 uplift.

Change-Id: Ib24ff0304b6279ff0be749141854d6a604473597
changes/15/689015/13
Ahmad Mahmoudi 2 years ago
parent
commit
f0f876afef
4 changed files with 25 additions and 11 deletions
  1. +4
    -0
      Makefile
  2. +11
    -7
      charts/shipyard/values.yaml
  3. +5
    -2
      images/airflow/Dockerfile.opensuse_15
  4. +5
    -2
      images/airflow/Dockerfile.ubuntu_xenial

+ 4
- 0
Makefile View File

@ -32,6 +32,7 @@ NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local
USE_PROXY ?= false
AIRFLOW_SRC ?=
AIRFLOW_HOME ?=
DISTRO_BASE_IMAGE ?=
DISTRO ?= ubuntu_xenial
@ -86,6 +87,7 @@ run:
_BASE_IMAGE_ARG := $(if $(DISTRO_BASE_IMAGE),--build-arg FROM="${DISTRO_BASE_IMAGE}" ,)
ifeq ($(IMAGE_NAME), airflow)
_AIRFLOW_SRC_ARG := $(if $(AIRFLOW_SRC),--build-arg AIRFLOW_SRC="${AIRFLOW_SRC}" ,)
_AIRFLOW_HOME_ARG := $(if $(AIRFLOW_HOME),--build-arg AIRFLOW_HOME="${AIRFLOW_HOME}" ,)
endif
.PHONY: build
@ -98,6 +100,7 @@ ifeq ($(USE_PROXY), true)
-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
$(_BASE_IMAGE_ARG) \
$(_AIRFLOW_SRC_ARG) \
$(_AIRFLOW_HOME_ARG) \
--build-arg http_proxy=$(PROXY) \
--build-arg https_proxy=$(PROXY) \
--build-arg HTTP_PROXY=$(PROXY) \
@ -113,6 +116,7 @@ else
-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
$(_BASE_IMAGE_ARG) \
$(_AIRFLOW_SRC_ARG) \
$(_AIRFLOW_HOME_ARG) \
--build-arg ctx_base=$(BUILD_CTX) .
endif
ifeq ($(PUSH_IMAGE), true)


+ 11
- 7
charts/shipyard/values.yaml View File

@ -454,7 +454,9 @@ conf:
# NOTE: Airflow 1.10 introduces a need to declare all config options:
# https://issues.apache.org/jira/browse/AIRFLOW-3099
core:
airflow_home: /usr/local/airflow
# core.airflow_home is not used in 1.10.3 and later.
# Envrionment variable AIRFLOW_HOME is used instead.
#airflow_home: /usr/local/airflow
dags_folder: /usr/local/airflow/dags
base_log_folder: /usr/local/airflow/logs
remote_logging: "False"
@ -497,15 +499,17 @@ conf:
parallelism: 32
dag_concurrency: 8
dags_are_paused_at_creation: "False"
non_pooled_task_slot_count: 128
# In 1.10.4, non_pooled_task_slot_count and non_pooled_backfill_task_slot_count are
# removed in favor of default_pool, which is initialized with 128 slots by default.
#non_pooled_task_slot_count: 128
max_active_runs_per_dag: 8
load_examples: "False"
plugins_folder: /usr/local/airflow/plugins
fernet_key: fKp7omMJ4QlTxfZzVBSiyXVgeCK-6epRjGgMpEIsjvs=
donot_pickle: "False"
dagbag_import_timeout: 30
task_runner: "BashTaskRunner"
# task_runner: "StandardTaskRunner" -- coming soon?
# BashTaskRunner has been renamed to StandardTaskRunner from 1.10.3
task_runner: "StandardTaskRunner"
default_impersonation: ""
security: ""
secure_mode: "True"
@ -683,9 +687,9 @@ conf:
hide_sensitive_variable_fields: "True"
elasticsearch:
# Shipyard is not using this
elasticsearch_host: ""
elasticsearch_log_id_template: ""
elasticsearch_end_of_log_mark: ""
host: ""
log_id_template: ""
end_of_log_mark: ""
kubernetes:
# Shipyard is not using this (maybe future for spawning own workers)
worker_container_repository: ""


+ 5
- 2
images/airflow/Dockerfile.opensuse_15 View File

@ -40,11 +40,14 @@ EXPOSE $WORKER_PORT
# Set ARG for usage during build
ARG AIRFLOW_HOME=/usr/local/airflow
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4"
ARG ctx_base=src/bin
# Kubectl version
ARG KUBECTL_VERSION=1.10.2
ARG KUBECTL_VERSION=1.16.2
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
RUN set -ex && \
zypper -q update -y ;\


+ 5
- 2
images/airflow/Dockerfile.ubuntu_xenial View File

@ -40,12 +40,15 @@ EXPOSE $WORKER_PORT
# Set ARG for usage during build
ARG AIRFLOW_HOME=/usr/local/airflow
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.4"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
# Kubectl version
ARG KUBECTL_VERSION=1.10.2
ARG KUBECTL_VERSION=1.16.2
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
RUN set -ex && \
apt-get -qq update && \


Loading…
Cancel
Save