Browse Source

(airflow) Configure source of Airflow pkg

- Allow a Docker build-arg to specify the source of the Airflow
  python package in any format supported by pip

Change-Id: Ifa2dd62d91570035cff91ff07868b0fcf659add8
changes/40/679840/5
Scott Hussey 2 years ago
committed by Mahmoudi, Ahmad (am495p)
parent
commit
185dc9823c
5 changed files with 17 additions and 3 deletions
  1. +6
    -0
      Makefile
  2. +3
    -1
      images/airflow/Dockerfile.opensuse_15
  3. +3
    -0
      images/airflow/Dockerfile.ubuntu_xenial
  4. +3
    -1
      images/airflow/requirements.txt
  5. +2
    -1
      src/bin/shipyard_airflow/requirements.txt

+ 6
- 0
Makefile View File

@ -31,6 +31,7 @@ PROXY ?= http://proxy.foo.com:8000
NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local
USE_PROXY ?= false
AIRFLOW_SRC ?=
DISTRO_BASE_IMAGE ?=
DISTRO ?= ubuntu_xenial
@ -83,6 +84,9 @@ run:
USE_PROXY=$(USE_PROXY) PROXY=$(PROXY) $(SCRIPT) $(IMAGE)
_BASE_IMAGE_ARG := $(if $(DISTRO_BASE_IMAGE),--build-arg FROM="${DISTRO_BASE_IMAGE}" ,)
ifeq ($(IMAGE_NAME), airflow)
_AIRFLOW_SRC_ARG := $(if $(AIRFLOW_SRC),--build-arg AIRFLOW_SRC="${AIRFLOW_SRC}" ,)
endif
.PHONY: build
build:
@ -93,6 +97,7 @@ ifeq ($(USE_PROXY), true)
--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
$(_BASE_IMAGE_ARG) \
$(_AIRFLOW_SRC_ARG) \
--build-arg http_proxy=$(PROXY) \
--build-arg https_proxy=$(PROXY) \
--build-arg HTTP_PROXY=$(PROXY) \
@ -107,6 +112,7 @@ else
--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
-f $(IMAGE_DIR)/Dockerfile.$(DISTRO) \
$(_BASE_IMAGE_ARG) \
$(_AIRFLOW_SRC_ARG) \
--build-arg ctx_base=$(BUILD_CTX) .
endif
ifeq ($(PUSH_IMAGE), true)


+ 3
- 1
images/airflow/Dockerfile.opensuse_15 View File

@ -40,6 +40,7 @@ EXPOSE $WORKER_PORT
# Set ARG for usage during build
ARG AIRFLOW_HOME=/usr/local/airflow
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
ARG ctx_base=src/bin
# Kubectl version
@ -110,7 +111,8 @@ ENV PBR_VERSION 0.1a1
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir \
&& pip3 install $AIRFLOW_SRC --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \


+ 3
- 0
images/airflow/Dockerfile.ubuntu_xenial View File

@ -40,6 +40,7 @@ EXPOSE $WORKER_PORT
# Set ARG for usage during build
ARG AIRFLOW_HOME=/usr/local/airflow
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
@ -85,8 +86,10 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
# Dependency requirements
# Note - removing snakebite (python 2 vs. 3). See:
# https://github.com/puckel/docker-airflow/issues/77
# Install Airflow directly to allow overriding source
COPY images/airflow/requirements.txt /tmp/
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
&& pip3 install $AIRFLOW_SRC --no-cache-dir \
&& pip3 uninstall -y snakebite || true
# Copy scripts used in the container:


+ 3
- 1
images/airflow/requirements.txt View File

@ -18,9 +18,11 @@ ndg-httpsclient==0.5.1
pyasn1==0.4.4
psycopg2==2.7.5
docker==3.5.0
apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1
# Airflow is now installed in the Dockerfile directory to allow for
# overriding where it is sourced from
python-openstackclient==3.16.1
kubernetes>=6.0.0
marshmallow-sqlalchemy==0.18.0
# Dependencies for other UCP components
git+https://git.openstack.org/openstack/airship-deckhand@a76ffb66ae809c19281a6cf5c9414ed197d249b7#egg=deckhand


+ 2
- 1
src/bin/shipyard_airflow/requirements.txt View File

@ -36,7 +36,8 @@ ulid==1.1
uwsgi==2.0.17
# To support profiling in non-prod
Werkzeug==0.14.1
Werkzeug>=0.15.0
marshmallow-sqlalchemy==0.18.0
# Dependencies for other UCP components
git+https://git.openstack.org/openstack/airship-deckhand@a76ffb66ae809c19281a6cf5c9414ed197d249b7#egg=deckhand


Loading…
Cancel
Save