Airflow 2.10.2 + ubuntu_jammy
Change-Id: I88d91ffc9fedb51a61cb34a77e6fd7b9ea187862
This commit is contained in:
Sergiy Markin
parent
c8be88cfa2
commit
e917c9dca7
137
.zuul.yaml
137
.zuul.yaml
@ -13,30 +13,34 @@
|
||||
- project:
|
||||
templates:
|
||||
- docs-on-readthedocs
|
||||
- openstack-python38-jobs
|
||||
vars:
|
||||
rtd_webhook_id: '38576'
|
||||
rtd_project_name: 'airship-shipyard'
|
||||
ensure_global_symlinks: true
|
||||
check:
|
||||
jobs:
|
||||
- airship-shipyard-openstack-tox-pep8-focal
|
||||
- airship-shipyard-openstack-tox-cover-focal
|
||||
- openstack-tox-py310
|
||||
- airship-shipyard-openstack-tox-pep8-jammy
|
||||
- airship-shipyard-openstack-tox-cover-jammy
|
||||
- airship-shipyard-chart-build-gate
|
||||
- airship-shipyard-chart-build-latest-htk
|
||||
- airship-airflow-dependency-vulnerability-check
|
||||
- airship-shipyard-dependency-vulnerability-check
|
||||
- airship-shipyard-whitespace-lint-gate
|
||||
- airship-shipyard-airskiff-deployment-focal
|
||||
- shipyard-airskiff-deployment-focal-kubeadm
|
||||
- airship-shipyard-image-gate-ubuntu_jammy
|
||||
- shipyard-airskiff-deployment-jammy-kubeadm
|
||||
gate:
|
||||
jobs:
|
||||
- airship-shipyard-openstack-tox-pep8-focal
|
||||
- airship-shipyard-openstack-tox-cover-focal
|
||||
- openstack-tox-py310
|
||||
- airship-shipyard-openstack-tox-pep8-jammy
|
||||
- airship-shipyard-openstack-tox-cover-jammy
|
||||
- airship-shipyard-chart-build-gate
|
||||
- airship-shipyard-whitespace-lint-gate
|
||||
- airship-shipyard-airskiff-deployment-focal
|
||||
- airship-shipyard-image-gate-ubuntu_jammy
|
||||
- shipyard-airskiff-deployment-jammy-kubeadm
|
||||
post:
|
||||
jobs:
|
||||
- airship-shipyard-docker-build-publish-ubuntu_focal
|
||||
- airship-shipyard-docker-build-publish-ubuntu_jammy
|
||||
- shipyard-upload-git-mirror
|
||||
|
||||
- nodeset:
|
||||
@ -46,32 +50,52 @@
|
||||
label: ubuntu-bionic
|
||||
|
||||
- nodeset:
|
||||
name: airship-shipyard-single-node-focal
|
||||
name: airship-shipyard-single-node-jammy
|
||||
nodes:
|
||||
- name: primary
|
||||
label: ubuntu-focal
|
||||
label: ubuntu-jammy
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-openstack-tox-pep8-focal
|
||||
name: airship-airflow-dependency-vulnerability-check
|
||||
parent: tox-py310
|
||||
voting: false
|
||||
timeout: 600
|
||||
nodeset: ubuntu-jammy
|
||||
vars:
|
||||
tox_envlist: airflow_safety
|
||||
bindep_profile: test py310
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-dependency-vulnerability-check
|
||||
parent: tox-py310
|
||||
voting: false
|
||||
timeout: 600
|
||||
nodeset: ubuntu-jammy
|
||||
vars:
|
||||
tox_envlist: shipyard_safety
|
||||
bindep_profile: test py310
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-openstack-tox-pep8-jammy
|
||||
parent: openstack-tox-pep8
|
||||
description: Runs pep8 job on focal
|
||||
nodeset: airship-shipyard-single-node-focal
|
||||
description: Runs pep8 job on jammy
|
||||
nodeset: airship-shipyard-single-node-jammy
|
||||
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-openstack-tox-cover-focal
|
||||
name: airship-shipyard-openstack-tox-cover-jammy
|
||||
parent: openstack-tox-cover
|
||||
description: Runs cover job on focal
|
||||
nodeset: airship-shipyard-single-node-focal
|
||||
description: Runs cover job on jammy
|
||||
nodeset: airship-shipyard-single-node-jammy
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-chart-build-gate
|
||||
description: Build charts using pinned Helm toolkit.
|
||||
timeout: 900
|
||||
run: tools/gate/playbooks/build-charts.yaml
|
||||
nodeset: airship-shipyard-single-node-focal
|
||||
nodeset: airship-shipyard-single-node-jammy
|
||||
vars:
|
||||
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
|
||||
HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-chart-build-latest-htk
|
||||
@ -79,7 +103,7 @@
|
||||
timeout: 900
|
||||
voting: false
|
||||
run: tools/gate/playbooks/build-charts.yaml
|
||||
nodeset: airship-shipyard-single-node-focal
|
||||
nodeset: airship-shipyard-single-node-jammy
|
||||
vars:
|
||||
HTK_COMMIT: master
|
||||
|
||||
@ -88,7 +112,7 @@
|
||||
description: |
|
||||
Lints all files by checking them for whitespace.
|
||||
run: tools/gate/playbooks/zuul-linter.yaml
|
||||
nodeset: airship-shipyard-single-node-focal
|
||||
nodeset: airship-shipyard-single-node-jammy
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-image-base
|
||||
@ -96,7 +120,7 @@
|
||||
Base job for running airship-shipyard image related jobs.
|
||||
timeout: 3600
|
||||
run: tools/gate/playbooks/docker-image-build.yaml
|
||||
nodeset: airship-shipyard-single-node-focal
|
||||
nodeset: airship-shipyard-single-node-jammy
|
||||
irrelevant-files: &irrelevant-files
|
||||
- ^.*\.rst$
|
||||
- ^doc/.*$
|
||||
@ -104,39 +128,17 @@
|
||||
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-image-gate-ubuntu_focal
|
||||
name: airship-shipyard-image-gate-ubuntu_jammy
|
||||
description: |
|
||||
Run shipyard-image build for ubuntu_focal.
|
||||
Run shipyard-image build for ubuntu_jammy.
|
||||
parent: airship-shipyard-image-base
|
||||
vars:
|
||||
publish: false
|
||||
distro: ubuntu_focal
|
||||
distro: ubuntu_jammy
|
||||
tags:
|
||||
dynamic:
|
||||
patch_set: true
|
||||
|
||||
# - job:
|
||||
# name: airship-shipyard-airskiff-deployment-focal
|
||||
# nodeset: airship-shipyard-single-node-focal
|
||||
# description: |
|
||||
# Deploy Memcached using Airskiff and submitted Armada changes.
|
||||
# timeout: 9600
|
||||
# voting: true
|
||||
# pre-run:
|
||||
# - tools/gate/playbooks/airskiff-reduce-site.yaml
|
||||
# - tools/gate/playbooks/git-config.yaml
|
||||
# run: tools/gate/playbooks/airskiff-deploy.yaml
|
||||
# post-run: tools/gate/playbooks/debug-report.yaml
|
||||
# required-projects:
|
||||
# - airship/treasuremap
|
||||
# vars:
|
||||
# CLONE_SHIPYARD: false
|
||||
# SHIPYARD_IMAGE_DISTRO: ubuntu_focal
|
||||
# HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
|
||||
# HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
|
||||
# OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
|
||||
# OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
|
||||
# irrelevant-files: *irrelevant-files
|
||||
|
||||
- job:
|
||||
name: shipyard-base
|
||||
@ -153,6 +155,7 @@
|
||||
irrelevant-files: *irrelevant-files
|
||||
timeout: 10800
|
||||
pre-run:
|
||||
- tools/gate/playbooks/checkout-treasuremap-ref.yaml
|
||||
- tools/gate/playbooks/prepare-hosts.yaml
|
||||
- tools/gate/playbooks/mount-volumes.yaml
|
||||
- tools/gate/playbooks/deploy-env.yaml
|
||||
@ -161,6 +164,7 @@
|
||||
post-run:
|
||||
- tools/gate/playbooks/osh-infra-collect-logs.yaml
|
||||
vars:
|
||||
treasuremap_ref: refs/changes/43/927643/4
|
||||
extra_volume:
|
||||
size: 80G
|
||||
type: Linux
|
||||
@ -177,16 +181,16 @@
|
||||
loopback_device: /dev/loop100
|
||||
loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
|
||||
ceph_osd_data_device: /dev/loop100
|
||||
kube_version_repo: "v1.29"
|
||||
# the list of k8s package versions are available here
|
||||
# https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages
|
||||
kube_version: "1.29.5-1.1"
|
||||
kube_version_repo: "v1.30"
|
||||
kube_version: "1.30.3-1.1"
|
||||
calico_setup: true
|
||||
calico_version: "v3.27.0"
|
||||
calico_version: "v3.27.4"
|
||||
cilium_setup: false
|
||||
helm_version: "v3.14.2"
|
||||
yq_version: "v4.6.0"
|
||||
crictl_version: "v1.26.1"
|
||||
cilium_version: "1.16.0"
|
||||
flannel_setup: false
|
||||
flannel_version: v0.25.4
|
||||
helm_version: "v3.15.4"
|
||||
crictl_version: "v1.30.1"
|
||||
zuul_osh_relative_path: ../../openstack/openstack-helm
|
||||
zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
|
||||
zuul_treasuremap_relative_path: ../../airship/treasuremap
|
||||
@ -194,19 +198,20 @@
|
||||
run_helm_tests: "no"
|
||||
|
||||
- job:
|
||||
name: shipyard-airskiff-deployment-focal-kubeadm
|
||||
nodeset: treasuremap-airskiff-1node-ubuntu_focal
|
||||
name: shipyard-airskiff-deployment-jammy-kubeadm
|
||||
nodeset: treasuremap-airskiff-1node-ubuntu_jammy
|
||||
description: |
|
||||
Deploy Memcached using Airskiff and submitted Shipyard changes.
|
||||
parent: shipyard-base
|
||||
vars:
|
||||
site: airskiff
|
||||
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.14.2-linux-amd64.tar.gz
|
||||
HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
|
||||
OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
|
||||
OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015
|
||||
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
|
||||
HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
|
||||
OSH_INFRA_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b
|
||||
OSH_COMMIT: 540df5cb0dbdaed63c202e2d6f2b7891062f8203
|
||||
TREASUREMAP_COMMIT: refs/changes/43/927643/4
|
||||
CLONE_SHIPYARD: false
|
||||
DISTRO: ubuntu_focal
|
||||
DISTRO: ubuntu_jammy
|
||||
DOCKER_REGISTRY: localhost:5000
|
||||
MAKE_SHIPYARD_IMAGES: true
|
||||
gate_scripts_relative_path: ../../airship/treasuremap
|
||||
@ -227,23 +232,23 @@
|
||||
irrelevant-files: *irrelevant-files
|
||||
|
||||
- job:
|
||||
name: airship-shipyard-docker-build-publish-ubuntu_focal
|
||||
name: airship-shipyard-docker-build-publish-ubuntu_jammy
|
||||
timeout: 3600
|
||||
run: tools/gate/playbooks/run-image.yaml
|
||||
nodeset: airship-shipyard-single-node-focal
|
||||
nodeset: airship-shipyard-single-node-jammy
|
||||
irrelevant-files: *irrelevant-files
|
||||
secrets:
|
||||
- airship_shipyard_quay_creds
|
||||
vars:
|
||||
publish: true
|
||||
distro: ubuntu_focal
|
||||
distro: ubuntu_jammy
|
||||
tags:
|
||||
dynamic:
|
||||
branch: true
|
||||
commit: true
|
||||
static:
|
||||
- latest
|
||||
- airflow_2.8.2
|
||||
- airflow_2.10.2
|
||||
|
||||
|
||||
|
||||
|
||||
11
Makefile
11
Makefile
@ -34,9 +34,11 @@ USE_PROXY ?= false
|
||||
AIRFLOW_SRC ?=
|
||||
AIRFLOW_HOME ?=
|
||||
DISTRO_BASE_IMAGE ?=
|
||||
DISTRO ?= ubuntu_focal
|
||||
DISTRO ?= ubuntu_jammy
|
||||
DISTRO_ALIAS ?= ubuntu_focal
|
||||
|
||||
IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO}
|
||||
IMAGE_ALIAS := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO_ALIAS}
|
||||
IMAGE_DIR:=images/$(IMAGE_NAME)
|
||||
|
||||
.PHONY: images
|
||||
@ -137,6 +139,13 @@ else
|
||||
$(_AIRFLOW_HOME_ARG) \
|
||||
--build-arg ctx_base=$(BUILD_CTX) .
|
||||
endif
|
||||
ifneq ($(DISTRO), $(DISTRO_ALIAS))
|
||||
docker tag $(IMAGE) $(IMAGE_ALIAS)
|
||||
endif
|
||||
ifeq ($(DOCKER_REGISTRY), localhost:5000)
|
||||
docker push $(IMAGE)
|
||||
docker push $(IMAGE_ALIAS)
|
||||
endif
|
||||
ifeq ($(PUSH_IMAGE), true)
|
||||
docker push $(IMAGE)
|
||||
endif
|
||||
|
||||
@ -16,8 +16,8 @@
|
||||
apiVersion: v1
|
||||
description: A Helm chart for Shipyard and Airflow
|
||||
name: shipyard
|
||||
version: 0.2.6
|
||||
appVersion: 2.8.2
|
||||
version: 0.2.7
|
||||
appVersion: 2.10.2
|
||||
keywords:
|
||||
- shipyard
|
||||
- airflow
|
||||
|
||||
@ -180,4 +180,4 @@ USER airflow
|
||||
WORKDIR ${AIRFLOW_HOME}
|
||||
|
||||
# Execute entrypoint
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
186
images/airflow/Dockerfile.ubuntu_jammy
Normal file
186
images/airflow/Dockerfile.ubuntu_jammy
Normal file
@ -0,0 +1,186 @@
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Docker image to run Airflow on Kubernetes
|
||||
ARG FROM=ubuntu:jammy
|
||||
FROM ${FROM}
|
||||
|
||||
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
|
||||
org.opencontainers.image.url='https://airshipit.org' \
|
||||
org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
|
||||
org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
|
||||
org.opencontainers.image.vendor='The Airship Authors' \
|
||||
org.opencontainers.image.licenses='Apache-2.0'
|
||||
|
||||
# Do not prompt user for choices on installation/configuration of packages
|
||||
# Set port 8080 for Airflow Web
|
||||
# Set port 5555 for Airflow Flower
|
||||
# Set port 8793 for Airflow Worker
|
||||
ENV container docker
|
||||
ENV WEB_PORT 8080
|
||||
ENV FLOWER_PORT 5555
|
||||
ENV WORKER_PORT 8793
|
||||
ENV SLUGIFY_USES_TEXT_UNIDECODE yes
|
||||
ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
|
||||
|
||||
# Expose port for applications
|
||||
EXPOSE $WEB_PORT
|
||||
EXPOSE $FLOWER_PORT
|
||||
EXPOSE $WORKER_PORT
|
||||
|
||||
# Set ARG for usage during build
|
||||
ARG AIRFLOW_HOME=/usr/local/airflow
|
||||
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
|
||||
# version of celery incompatibile with the version of kombu needed by other
|
||||
# Airship components
|
||||
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2"
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG ctx_base=src/bin
|
||||
|
||||
# Kubectl version
|
||||
ARG KUBECTL_VERSION=1.31.0
|
||||
|
||||
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
|
||||
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
|
||||
|
||||
RUN set -ex \
|
||||
&& apt-get update && apt-get upgrade -y \
|
||||
&& apt-get install -y wget curl \
|
||||
apt-transport-https ca-certificates gnupg \
|
||||
&& echo "deb http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
|
||||
&& curl -o /etc/apt/trusted.gpg.d/postgresql_release_signing_key.asc 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' \
|
||||
&& apt update \
|
||||
&& apt -y install \
|
||||
automake \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
g++ \
|
||||
libkrb5-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
libpq-dev \
|
||||
libpq5 \
|
||||
libtool \
|
||||
libxml2 \
|
||||
libsasl2-dev \
|
||||
locales \
|
||||
netcat \
|
||||
netbase \
|
||||
postgresql-client \
|
||||
postgresql-common \
|
||||
python3 \
|
||||
python3-setuptools \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
python3-dateutil \
|
||||
make \
|
||||
--no-install-recommends \
|
||||
&& python3 -m pip install -U pip \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/tmp/* \
|
||||
/var/tmp/* \
|
||||
/usr/share/man \
|
||||
/usr/share/doc \
|
||||
/usr/share/doc-base \
|
||||
/etc/ssl/private/ssl-cert-snakeoil.key \
|
||||
/etc/ssl/private/ssl-cert-snakeoil.pem
|
||||
|
||||
# Install LibYAML
|
||||
ENV LD_LIBRARY_PATH=/usr/local/lib
|
||||
|
||||
ARG LIBYAML_VERSION=0.2.5
|
||||
RUN set -ex \
|
||||
&& git clone https://github.com/yaml/libyaml.git \
|
||||
&& cd libyaml \
|
||||
&& git checkout $LIBYAML_VERSION \
|
||||
&& ./bootstrap \
|
||||
&& ./configure \
|
||||
&& make \
|
||||
&& make install \
|
||||
&& cd .. \
|
||||
&& rm -fr libyaml
|
||||
|
||||
|
||||
# Things that change mostly infrequently
|
||||
RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
|
||||
&& curl -L -o /usr/local/bin/kubectl \
|
||||
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
|
||||
&& chmod +x /usr/local/bin/kubectl
|
||||
|
||||
# Dependency requirements
|
||||
# Note - removing snakebite (python 2 vs. 3). See:
|
||||
# https://github.com/puckel/docker-airflow/issues/77
|
||||
# Install Airflow directly to allow overriding source
|
||||
# COPY images/airflow/requirements.txt /tmp/
|
||||
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt
|
||||
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
|
||||
&& pip3 install $AIRFLOW_SRC --no-cache-dir
|
||||
# && (pip3 uninstall -y snakebite || true) \
|
||||
# && (pip3 uninstall -y psycopg2 || true) \
|
||||
# && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true)
|
||||
|
||||
# Copy scripts used in the container:
|
||||
COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/
|
||||
|
||||
# Copy configuration (e.g. logging config for Airflow):
|
||||
COPY images/airflow/config/*.py ${AIRFLOW_HOME}/config/
|
||||
COPY images/airflow/webserver_config.py ${AIRFLOW_HOME}/
|
||||
|
||||
# Change permissions
|
||||
RUN chown -R airflow: ${AIRFLOW_HOME}
|
||||
|
||||
# Setting the version explicitly for PBR
|
||||
ENV PBR_VERSION 0.1a1
|
||||
|
||||
# Shipyard
|
||||
#
|
||||
# Shipyard provides core functionality used by the Airflow plugins/operators
|
||||
# Since Shipyard and Airflow are built together as images, this should prevent
|
||||
# stale or out-of-date code between these parts.
|
||||
# Shipyard requirements, source and installation
|
||||
# COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
|
||||
# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
|
||||
|
||||
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
|
||||
RUN cd /tmp/shipyard \
|
||||
&& pip3 install $(pwd) --use-pep517
|
||||
|
||||
# Note: The value for the dags and plugins directories that are sourced
|
||||
# from the values.yaml of the Shipyard Helm chart need to align with these
|
||||
# directories. If they do not, airflow will not find the intended dags and
|
||||
# plugins.
|
||||
#
|
||||
# Note: In the case of building images using the provided Makefile, a test is
|
||||
# run against the built-in dags provided with Airflow. Since there is no Helm
|
||||
# chart to reconfigure the airflow.cfg with these directories, these dags and
|
||||
# plugins are not known to Airflow during the image test.
|
||||
#
|
||||
# Copy the plugins and dags that will be used by this Airflow image:
|
||||
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
|
||||
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins \
|
||||
/usr/local/lib/python3.10/dist-packages/airflow/plugins/
|
||||
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
|
||||
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags \
|
||||
/usr/local/lib/python3.10/dist-packages/airflow/dags/
|
||||
|
||||
# Set work directory
|
||||
USER airflow
|
||||
WORKDIR ${AIRFLOW_HOME}
|
||||
|
||||
# Execute entrypoint
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
@ -40,13 +40,26 @@ elif [[ $cmd == *scheduler* ]]; then
|
||||
${python3_path} ${airflow_path} scheduler $2 $3
|
||||
done
|
||||
elif [[ $cmd == 'quicktest' ]]; then
|
||||
${python3_path} ${airflow_path} version
|
||||
${python3_path} ${airflow_path} db init
|
||||
${python3_path} ${airflow_path} db migrate
|
||||
${python3_path} ${airflow_path} dags list
|
||||
${python3_path} ${airflow_path} webserver -p 8080 &
|
||||
${python3_path} ${airflow_path} scheduler &
|
||||
while true; do
|
||||
is_active=$(airflow dags details example_bash_operator -o plain | grep is_active | awk '{print $2}')
|
||||
if [ "$is_active" == "True" ]; then
|
||||
echo "DAG example_bash_operator is active."
|
||||
break
|
||||
fi
|
||||
echo "Waiting for DAG example_bash_operator to become active..."
|
||||
sleep 10
|
||||
done
|
||||
${python3_path} ${airflow_path} dags list
|
||||
${python3_path} ${airflow_path} dags unpause example_bash_operator
|
||||
${python3_path} ${airflow_path} tasks test example_bash_operator runme_0
|
||||
${python3_path} ${airflow_path} dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
|
||||
${python3_path} ${airflow_path} tasks run example_bash_operator runme_0 2018-01-01
|
||||
${python3_path} ${airflow_path} tasks states-for-dag-run example_bash_operator backfill__2018-01-02T00:00:00+00:00
|
||||
${python3_path} ${airflow_path} dags state example_bash_operator 2018-01-01
|
||||
else
|
||||
echo "Invalid Command!"
|
||||
|
||||
141
images/shipyard/Dockerfile.ubuntu_jammy
Normal file
141
images/shipyard/Dockerfile.ubuntu_jammy
Normal file
@ -0,0 +1,141 @@
|
||||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
ARG FROM=ubuntu:jammy
|
||||
FROM ${FROM}
|
||||
|
||||
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
|
||||
org.opencontainers.image.url='https://airshipit.org' \
|
||||
org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \
|
||||
org.opencontainers.image.source='https://opendev.org/airship/shipyard' \
|
||||
org.opencontainers.image.vendor='The Airship Authors' \
|
||||
org.opencontainers.image.licenses='Apache-2.0'
|
||||
|
||||
ENV container docker
|
||||
ENV PORT 9000
|
||||
ENV LC_ALL C.UTF-8
|
||||
ENV LANG C.UTF-8
|
||||
|
||||
ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
|
||||
|
||||
# Setting the version explicitly for PBR
|
||||
ENV PBR_VERSION 0.1a1
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG ctx_base=src/bin
|
||||
|
||||
# Expose port 9000 for application
|
||||
EXPOSE $PORT
|
||||
|
||||
RUN set -ex && \
|
||||
apt-get update && apt-get upgrade -y && \
|
||||
apt-get -y install \
|
||||
automake \
|
||||
ca-certificates \
|
||||
curl \
|
||||
build-essential \
|
||||
git \
|
||||
libtool \
|
||||
libxml2 \
|
||||
make \
|
||||
netbase \
|
||||
python3-dev \
|
||||
python3-setuptools \
|
||||
--no-install-recommends \
|
||||
&& apt-get autoremove -yqq --purge \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/tmp/* \
|
||||
/var/tmp/* \
|
||||
/usr/share/man \
|
||||
/usr/share/doc \
|
||||
/usr/share/doc-base
|
||||
|
||||
# Install LibYAML
|
||||
ENV LD_LIBRARY_PATH=/usr/local/lib
|
||||
|
||||
ARG LIBYAML_VERSION=0.2.5
|
||||
RUN set -ex \
|
||||
&& git clone https://github.com/yaml/libyaml.git \
|
||||
&& cd libyaml \
|
||||
&& git checkout $LIBYAML_VERSION \
|
||||
&& ./bootstrap \
|
||||
&& ./configure \
|
||||
&& make \
|
||||
&& make install \
|
||||
&& cd .. \
|
||||
&& rm -fr libyaml
|
||||
|
||||
# Create shipyard user
|
||||
RUN useradd -ms /bin/bash shipyard \
|
||||
&& mkdir -p /home/shipyard/shipyard \
|
||||
&& mkdir -p /home/shipyard/shipyard_client
|
||||
|
||||
# Copy entrypoint.sh to /home/shipyard
|
||||
COPY ${ctx_base}/shipyard_airflow/entrypoint.sh /home/shipyard/entrypoint.sh
|
||||
# Change permissions and set up directories
|
||||
RUN chown -R shipyard: /home/shipyard \
|
||||
&& chmod +x /home/shipyard/entrypoint.sh
|
||||
|
||||
# Requirements and Shipyard source
|
||||
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /home/shipyard/api_requirements.txt
|
||||
COPY ${ctx_base}/shipyard_client/requirements-frozen.txt /home/shipyard/client_requirements.txt
|
||||
COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
|
||||
COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
|
||||
|
||||
# Build
|
||||
RUN set -ex \
|
||||
&& buildDeps=' \
|
||||
build-essential \
|
||||
git \
|
||||
libssl-dev \
|
||||
libffi-dev \
|
||||
libkrb5-dev \
|
||||
libpq-dev \
|
||||
libsasl2-dev \
|
||||
libssl-dev \
|
||||
libre2-dev \
|
||||
libzmq3-dev \
|
||||
make \
|
||||
python3-pip \
|
||||
' \
|
||||
&& apt-get -qq update \
|
||||
&& apt-get upgrade -y \
|
||||
&& apt-get -y install -y $buildDeps --no-install-recommends \
|
||||
&& python3 -m pip install -U pip \
|
||||
&& pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
|
||||
&& cd /home/shipyard/shipyard_client \
|
||||
&& pip3 install . --use-pep517 --no-cache-dir \
|
||||
&& pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \
|
||||
&& cd /home/shipyard/shipyard \
|
||||
&& pip3 install . --use-pep517 --no-cache-dir \
|
||||
&& apt-get purge -y --auto-remove $buildDeps \
|
||||
&& apt-get autoremove -yqq --purge \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/tmp/* \
|
||||
/var/tmp/* \
|
||||
/usr/share/man \
|
||||
/usr/share/doc \
|
||||
/usr/share/doc-base \
|
||||
&& python3 -m pip install -U pip
|
||||
|
||||
|
||||
# Entrypoint
|
||||
ENTRYPOINT ["/home/shipyard/entrypoint.sh"]
|
||||
CMD ["server"]
|
||||
# Set user to shipyard
|
||||
USER shipyard
|
||||
@ -1,5 +1,5 @@
|
||||
alembic
|
||||
apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.8.2
|
||||
apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2
|
||||
arrow
|
||||
celery
|
||||
Flask
|
||||
@ -59,7 +59,7 @@ oslo.versionedobjects==3.1.0
|
||||
|
||||
|
||||
# Airship dependencies
|
||||
git+https://opendev.org/airship/deckhand.git@32e9950db405b5d1eb74da0c4e8df344a8638eff#egg=deckhand
|
||||
git+https://opendev.org/airship/drydock.git@6b317dda0518e3366c9835ee99d9691ffba718eb#egg=drydock_provisioner&subdirectory=python
|
||||
git+https://opendev.org/airship/armada.git@e85f0ccdeb93475aef2f2f4ce3da9f53a00309cd#egg=armada
|
||||
git+https://opendev.org/airship/promenade.git@d21e3434f641e1f1e7400a44e77fccb5b87a2824#egg=promenade
|
||||
git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6#egg=deckhand
|
||||
git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#egg=drydock_provisioner&subdirectory=python
|
||||
git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211#egg=armada
|
||||
git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66#egg=promenade
|
||||
|
||||
@ -1,158 +1,164 @@
|
||||
aiohttp==3.9.3
|
||||
aiohappyeyeballs==2.4.0
|
||||
aiohttp==3.10.5
|
||||
aiosignal==1.3.1
|
||||
alembic==1.13.1
|
||||
alembic==1.13.2
|
||||
amqp==5.2.0
|
||||
anyio==4.3.0
|
||||
apache-airflow==2.8.2
|
||||
apache-airflow-providers-apache-hdfs==4.3.2
|
||||
apache-airflow-providers-apache-hive==7.0.0
|
||||
apache-airflow-providers-celery==3.6.0
|
||||
apache-airflow-providers-common-io==1.3.0
|
||||
apache-airflow-providers-common-sql==1.11.0
|
||||
apache-airflow-providers-ftp==3.7.0
|
||||
apache-airflow-providers-http==4.9.1
|
||||
apache-airflow-providers-imap==3.5.0
|
||||
apache-airflow-providers-jdbc==4.2.2
|
||||
apache-airflow-providers-postgres==5.10.1
|
||||
apache-airflow-providers-sqlite==3.7.1
|
||||
apispec==6.4.0
|
||||
argcomplete==3.2.2
|
||||
Armada @ git+https://opendev.org/airship/armada.git@6f06dc09606e94bd6e7322a6d71c1ce5fdd33b0b
|
||||
anyio==4.4.0
|
||||
apache-airflow==2.10.2
|
||||
apache-airflow-providers-apache-hdfs==4.5.0
|
||||
apache-airflow-providers-apache-hive==8.2.0
|
||||
apache-airflow-providers-celery==3.8.1
|
||||
apache-airflow-providers-common-compat==1.2.0
|
||||
apache-airflow-providers-common-io==1.4.0
|
||||
apache-airflow-providers-common-sql==1.16.0
|
||||
apache-airflow-providers-fab==1.3.0
|
||||
apache-airflow-providers-ftp==3.11.0
|
||||
apache-airflow-providers-http==4.13.0
|
||||
apache-airflow-providers-imap==3.7.0
|
||||
apache-airflow-providers-jdbc==4.5.0
|
||||
apache-airflow-providers-postgres==5.12.0
|
||||
apache-airflow-providers-smtp==1.8.0
|
||||
apache-airflow-providers-sqlite==3.9.0
|
||||
apispec==6.6.1
|
||||
argcomplete==3.5.0
|
||||
Armada @ git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211
|
||||
arrow==1.3.0
|
||||
asgiref==3.7.2
|
||||
asgiref==3.8.1
|
||||
async-timeout==4.0.3
|
||||
attrs==23.2.0
|
||||
attrs==24.2.0
|
||||
autopage==0.5.2
|
||||
Babel==2.14.0
|
||||
backports.zoneinfo==0.2.1
|
||||
babel==2.16.0
|
||||
barbican==16.0.0
|
||||
bcrypt==4.1.2
|
||||
Beaker==1.12.1
|
||||
bcrypt==4.2.0
|
||||
Beaker==1.13.0
|
||||
billiard==4.2.0
|
||||
blinker==1.7.0
|
||||
blinker==1.8.2
|
||||
cachelib==0.9.0
|
||||
cachetools==5.3.2
|
||||
castellan==4.4.0
|
||||
celery==5.3.6
|
||||
certifi==2024.2.2
|
||||
cffi==1.16.0
|
||||
cachetools==5.5.0
|
||||
castellan==5.1.1
|
||||
celery==5.4.0
|
||||
certifi==2024.8.30
|
||||
cffi==1.17.1
|
||||
charset-normalizer==3.3.2
|
||||
click==8.1.7
|
||||
click-didyoumean==0.3.0
|
||||
click-didyoumean==0.3.1
|
||||
click-plugins==1.1.1
|
||||
click-repl==0.3.0
|
||||
clickclick==20.10.2
|
||||
cliff==4.6.0
|
||||
cliff==4.7.0
|
||||
cmd2==2.4.3
|
||||
colorama==0.4.6
|
||||
colorlog==4.8.0
|
||||
colorlog==6.8.2
|
||||
ConfigUpdater==3.2
|
||||
connexion==2.14.2
|
||||
cron-descriptor==1.4.3
|
||||
croniter==2.0.1
|
||||
cryptography==41.0.7
|
||||
cron-descriptor==1.4.5
|
||||
croniter==3.0.3
|
||||
cryptography==42.0.8
|
||||
debtcollector==3.0.0
|
||||
Deckhand @ git+https://opendev.org/airship/deckhand.git@fd58230f6e31ede4925bab0325105eb5b05ad1a8
|
||||
Deckhand @ git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6
|
||||
decorator==5.1.1
|
||||
deepdiff==6.7.1
|
||||
deepdiff==8.0.1
|
||||
Deprecated==1.2.14
|
||||
dill==0.3.1.1
|
||||
dnspython==2.6.1
|
||||
docopt==0.6.2
|
||||
docutils==0.20.1
|
||||
dogpile.cache==1.3.2
|
||||
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@816a4bbe366e2c1d58530986df05f99d7d9a7beb#subdirectory=python
|
||||
email-validator==1.3.1
|
||||
eventlet==0.35.2
|
||||
exceptiongroup==1.2.0
|
||||
docutils==0.16
|
||||
dogpile.cache==1.3.3
|
||||
drydock_provisioner @ git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#subdirectory=python
|
||||
email_validator==2.2.0
|
||||
eventlet==0.37.0
|
||||
exceptiongroup==1.2.2
|
||||
falcon==3.1.3
|
||||
fastavro==1.9.4
|
||||
fastavro==1.9.7
|
||||
fasteners==0.19
|
||||
fixtures==4.1.0
|
||||
Flask==2.2.5
|
||||
Flask-AppBuilder==4.3.11
|
||||
Flask-AppBuilder==4.5.0
|
||||
Flask-Babel==2.0.0
|
||||
Flask-Caching==2.1.0
|
||||
Flask-Caching==2.3.0
|
||||
Flask-JWT-Extended==4.6.0
|
||||
Flask-Limiter==3.5.1
|
||||
Flask-Limiter==3.8.0
|
||||
Flask-Login==0.6.3
|
||||
Flask-Session==0.5.0
|
||||
Flask-SQLAlchemy==2.5.1
|
||||
Flask-WTF==1.2.1
|
||||
flower==2.0.1
|
||||
frozenlist==1.4.1
|
||||
fsspec==2024.2.0
|
||||
fsspec==2024.9.0
|
||||
future==1.0.0
|
||||
futurist==3.0.0
|
||||
gitdb==4.0.11
|
||||
GitPython==3.1.42
|
||||
google-auth==2.28.1
|
||||
google-re2==1.1
|
||||
googleapis-common-protos==1.62.0
|
||||
graphviz==0.20.1
|
||||
greenlet==3.0.3
|
||||
grpcio==1.62.0
|
||||
GitPython==3.1.43
|
||||
google-auth==2.34.0
|
||||
google-re2==1.1.20240702
|
||||
googleapis-common-protos==1.65.0
|
||||
graphviz==0.20.3
|
||||
greenlet==3.1.0
|
||||
grpcio==1.66.1
|
||||
gssapi==1.8.3
|
||||
gunicorn==21.2.0
|
||||
gunicorn==23.0.0
|
||||
h11==0.14.0
|
||||
hdfs==2.7.3
|
||||
hmsclient==0.1.1
|
||||
html5lib==0.9999999
|
||||
httpcore==0.16.3
|
||||
httpcore==1.0.5
|
||||
httpexceptor==1.4.0
|
||||
httpx==0.23.3
|
||||
humanize==4.9.0
|
||||
idna==3.6
|
||||
httpx==0.27.0
|
||||
humanize==4.10.0
|
||||
idna==3.10
|
||||
importlib-metadata==6.11.0
|
||||
importlib-resources==5.13.0
|
||||
importlib_resources==6.4.5
|
||||
inflection==0.5.1
|
||||
iso8601==2.1.0
|
||||
itsdangerous==2.1.2
|
||||
itsdangerous==2.2.0
|
||||
JayDeBeApi==1.2.3
|
||||
Jinja2==3.1.3
|
||||
Jinja2==3.1.4
|
||||
jmespath==0.10.0
|
||||
JPype1==1.5.0
|
||||
jsonpath-ng==1.6.1
|
||||
jsonpickle==3.0.3
|
||||
jsonschema==4.21.1
|
||||
jsonpickle==3.3.0
|
||||
jsonschema==4.23.0
|
||||
jsonschema-specifications==2023.12.1
|
||||
keystoneauth1==5.1.2
|
||||
keystonemiddleware==10.2.0
|
||||
kombu==5.3.5
|
||||
krb5==0.5.1
|
||||
kubernetes==29.0.0
|
||||
kombu==5.4.1
|
||||
krb5==0.6.0
|
||||
kubernetes==30.1.0
|
||||
lazy-object-proxy==1.10.0
|
||||
ldap3==2.9.1
|
||||
limits==3.9.0
|
||||
limits==3.13.0
|
||||
linkify-it-py==2.0.3
|
||||
lockfile==0.12.2
|
||||
logutils==0.3.5
|
||||
Mako==1.3.2
|
||||
Mako==1.3.5
|
||||
markdown-it-py==3.0.0
|
||||
MarkupSafe==2.1.5
|
||||
marshmallow==3.20.2
|
||||
marshmallow==3.22.0
|
||||
marshmallow-oneofschema==3.1.1
|
||||
marshmallow-sqlalchemy==0.26.1
|
||||
mdit-py-plugins==0.4.0
|
||||
marshmallow-sqlalchemy==0.28.2
|
||||
mdit-py-plugins==0.4.2
|
||||
mdurl==0.1.2
|
||||
microversion-parse==1.0.1
|
||||
methodtools==0.4.7
|
||||
microversion-parse==2.0.0
|
||||
mock==5.1.0
|
||||
more-itertools==10.2.0
|
||||
msgpack==1.0.7
|
||||
multidict==6.0.5
|
||||
netaddr==1.2.1
|
||||
more-itertools==10.5.0
|
||||
msgpack==1.1.0
|
||||
multidict==6.1.0
|
||||
netaddr==1.3.0
|
||||
netifaces==0.11.0
|
||||
networkx==3.1
|
||||
numpy==1.24.4
|
||||
networkx==3.3
|
||||
numpy==1.26.4
|
||||
oauthlib==3.2.2
|
||||
opentelemetry-api==1.23.0
|
||||
opentelemetry-exporter-otlp==1.23.0
|
||||
opentelemetry-exporter-otlp-proto-common==1.23.0
|
||||
opentelemetry-exporter-otlp-proto-grpc==1.23.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.23.0
|
||||
opentelemetry-proto==1.23.0
|
||||
opentelemetry-sdk==1.23.0
|
||||
opentelemetry-semantic-conventions==0.44b0
|
||||
opentelemetry-api==1.27.0
|
||||
opentelemetry-exporter-otlp==1.27.0
|
||||
opentelemetry-exporter-otlp-proto-common==1.27.0
|
||||
opentelemetry-exporter-otlp-proto-grpc==1.27.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.27.0
|
||||
opentelemetry-proto==1.27.0
|
||||
opentelemetry-sdk==1.27.0
|
||||
opentelemetry-semantic-conventions==0.48b0
|
||||
ordered-set==4.1.0
|
||||
orderly-set==5.2.2
|
||||
os-service-types==1.7.0
|
||||
oslo.cache==3.3.1
|
||||
oslo.concurrency==5.1.1
|
||||
@ -170,115 +176,116 @@ oslo.service==3.1.1
|
||||
oslo.upgradecheck==2.1.1
|
||||
oslo.utils==6.1.0
|
||||
oslo.versionedobjects==3.1.0
|
||||
packaging==23.2
|
||||
pandas==2.0.3
|
||||
Paste==3.7.1
|
||||
packaging==24.1
|
||||
pandas==2.1.4
|
||||
Paste==3.10.1
|
||||
PasteDeploy==3.1.0
|
||||
PasteScript==3.4.0
|
||||
PasteScript==3.6.0
|
||||
pathspec==0.12.1
|
||||
pbr==6.0.0
|
||||
pbr==6.1.0
|
||||
pecan==1.5.1
|
||||
pendulum==3.0.0
|
||||
pip==23.2.1
|
||||
pkgutil_resolve_name==1.3.10
|
||||
pluggy==1.4.0
|
||||
pip==24.1
|
||||
pluggy==1.5.0
|
||||
ply==3.11
|
||||
prettytable==3.10.0
|
||||
prettytable==3.11.0
|
||||
prison==0.2.1
|
||||
promenade @ git+https://opendev.org/airship/promenade.git@14375e658bc06328042b1dc934eefcebf60210fd
|
||||
promenade @ git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66
|
||||
prometheus_client==0.20.0
|
||||
prompt-toolkit==3.0.43
|
||||
protobuf==4.25.3
|
||||
psutil==5.9.8
|
||||
prompt_toolkit==3.0.47
|
||||
protobuf==4.25.4
|
||||
psutil==6.0.0
|
||||
psycopg2-binary==2.9.9
|
||||
pure-sasl==0.6.2
|
||||
py==1.11.0
|
||||
pyarrow==14.0.2
|
||||
pyasn1==0.5.1
|
||||
pyasn1-modules==0.3.0
|
||||
pyarrow==16.1.0
|
||||
pyasn1==0.6.1
|
||||
pyasn1_modules==0.4.0
|
||||
pycadf==3.1.1
|
||||
pycparser==2.21
|
||||
Pygments==2.17.2
|
||||
pycparser==2.22
|
||||
Pygments==2.18.0
|
||||
PyHive==0.7.0
|
||||
PyJWT==2.8.0
|
||||
PyJWT==2.9.0
|
||||
pylibyaml==0.1.0
|
||||
pyOpenSSL==24.0.0
|
||||
pyparsing==3.1.1
|
||||
pyperclip==1.8.2
|
||||
pyspnego==0.10.2
|
||||
pyOpenSSL==24.2.1
|
||||
pyparsing==3.1.4
|
||||
pyperclip==1.9.0
|
||||
pyspnego==0.11.1
|
||||
python-barbicanclient==5.5.0
|
||||
python-daemon==3.0.1
|
||||
python-dateutil==2.8.2
|
||||
python-dateutil==2.9.0.post0
|
||||
python-keystoneclient==5.1.0
|
||||
python-memcached==1.62
|
||||
python-mimeparse==1.6.0
|
||||
python-nvd3==0.15.0
|
||||
python-mimeparse==2.0.0
|
||||
python-nvd3==0.16.0
|
||||
python-slugify==8.0.4
|
||||
python3-memcached==1.51
|
||||
pytz==2024.1
|
||||
PyYAML==6.0.1
|
||||
referencing==0.33.0
|
||||
regex==2023.12.25
|
||||
pytz==2024.2
|
||||
PyYAML==6.0.2
|
||||
redis==5.0.8
|
||||
referencing==0.35.1
|
||||
regex==2024.9.11
|
||||
repoze.lru==0.7
|
||||
requests==2.31.0
|
||||
requests-kerberos==0.14.0
|
||||
requests==2.32.3
|
||||
requests-kerberos==0.15.0
|
||||
requests-oauthlib==1.3.1
|
||||
requests-toolbelt==1.0.0
|
||||
resolver==0.2.1
|
||||
responses==0.25.0
|
||||
responses==0.25.3
|
||||
retry==0.9.2
|
||||
rfc3339-validator==0.1.4
|
||||
rfc3986==1.5.0
|
||||
rich==13.7.0
|
||||
rich-argparse==1.4.0
|
||||
rfc3986==2.0.0
|
||||
rich==13.8.1
|
||||
rich-argparse==1.5.2
|
||||
Routes==2.5.1
|
||||
rpds-py==0.18.0
|
||||
rpds-py==0.20.0
|
||||
rsa==4.9
|
||||
selector==0.10.1
|
||||
setproctitle==1.3.3
|
||||
setuptools==68.2.2
|
||||
simplejson==3.19.2
|
||||
setuptools==70.1.0
|
||||
simplejson==3.19.3
|
||||
six==1.16.0
|
||||
smmap==5.0.1
|
||||
sniffio==1.3.0
|
||||
SQLAlchemy==1.4.51
|
||||
sniffio==1.3.1
|
||||
SQLAlchemy==1.4.54
|
||||
SQLAlchemy-JSONField==1.0.2
|
||||
sqlalchemy-migrate==0.13.0
|
||||
SQLAlchemy-Utils==0.41.1
|
||||
sqlparse==0.4.4
|
||||
SQLAlchemy-Utils==0.41.2
|
||||
sqlparse==0.5.1
|
||||
statsd==4.0.1
|
||||
stevedore==5.2.0
|
||||
stevedore==5.3.0
|
||||
tabulate==0.9.0
|
||||
Tempita==0.5.2
|
||||
tenacity==8.2.3
|
||||
tenacity==8.5.0
|
||||
termcolor==2.4.0
|
||||
testresources==2.0.1
|
||||
testscenarios==0.5.0
|
||||
testtools==2.7.1
|
||||
testtools==2.7.2
|
||||
text-unidecode==1.3
|
||||
thrift==0.16.0
|
||||
thrift-sasl==0.4.3
|
||||
tiddlyweb==2.4.3
|
||||
time-machine==2.13.0
|
||||
tornado==6.4
|
||||
types-python-dateutil==2.8.19.20240106
|
||||
typing_extensions==4.9.0
|
||||
time-machine==2.15.0
|
||||
tornado==6.4.1
|
||||
types-python-dateutil==2.9.0.20240906
|
||||
typing_extensions==4.12.2
|
||||
tzdata==2024.1
|
||||
uc-micro-py==1.0.3
|
||||
ulid==1.1
|
||||
unicodecsv==0.14.1
|
||||
universal-pathlib==0.1.4
|
||||
urllib3==1.26.18
|
||||
uWSGI==2.0.24
|
||||
universal_pathlib==0.2.5
|
||||
urllib3==2.2.2
|
||||
uWSGI==2.0.27
|
||||
vine==5.1.0
|
||||
wcwidth==0.2.13
|
||||
WebOb==1.8.7
|
||||
websocket-client==1.7.0
|
||||
WebOb==1.8.8
|
||||
websocket-client==1.8.0
|
||||
Werkzeug==2.2.3
|
||||
wheel==0.41.2
|
||||
wheel==0.43.0
|
||||
wirerope==0.4.7
|
||||
wrapt==1.16.0
|
||||
WTForms==3.1.2
|
||||
xattr==0.10.1
|
||||
yappi==1.6.0
|
||||
yarl==1.9.4
|
||||
zipp==3.17.0
|
||||
yarl==1.11.1
|
||||
zipp==3.20.2
|
||||
|
||||
@ -19,20 +19,24 @@ import yaml
|
||||
import kubernetes
|
||||
|
||||
import airflow
|
||||
import pendulum
|
||||
from shipyard_airflow.common.document_validators import \
|
||||
document_validation_utils
|
||||
from shipyard_airflow.common.document_validators import errors
|
||||
from shipyard_airflow.plugins import deckhand_client_factory
|
||||
from shipyard_airflow.plugins import deployment_status_operator
|
||||
from shipyard_airflow.plugins import xcom_puller
|
||||
from airflow import DAG
|
||||
from airflow.models import TaskInstance, DagRun
|
||||
from airflow.utils.dates import days_ago
|
||||
from airflow.utils.state import State
|
||||
from airflow.utils.types import DagRunType
|
||||
|
||||
|
||||
@mock.patch.object(airflow.models.BaseOperator, '__init__')
|
||||
# @mock.patch.object(airflow.models.BaseOperator, '__init__')
|
||||
class TestDeploymentStatusOperator(TestCase):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(TestDeploymentStatusOperator, self).__init__(*args, **kwargs)
|
||||
self.context = {'ti': None}
|
||||
self.status = {'status': "doc"}
|
||||
self.revision_id = "revision_id"
|
||||
self.version = {'version': "doc"}
|
||||
@ -42,33 +46,51 @@ class TestDeploymentStatusOperator(TestCase):
|
||||
}
|
||||
self.config_map_data = {'release': yaml.safe_dump(full_data)}
|
||||
|
||||
@mock.patch.object(xcom_puller.XcomPuller, 'get_concurrency_status',
|
||||
return_value=True)
|
||||
def setUp(self):
|
||||
# Create a DAG for the test using pendulum
|
||||
self.dag = DAG(dag_id="example_dag", start_date=pendulum.now('UTC').add(days=-1))
|
||||
|
||||
# Create a DagRun to associate with the TaskInstance
|
||||
self.dag_run = DagRun(
|
||||
dag_id=self.dag.dag_id,
|
||||
execution_date=pendulum.now('UTC').add(days=-1), # Use pendulum for execution_date
|
||||
run_id="manual__" + pendulum.now('UTC').to_iso8601_string(),
|
||||
run_type=DagRunType.MANUAL,
|
||||
state=State.RUNNING
|
||||
)
|
||||
|
||||
# Create a TaskInstance with run_id
|
||||
self.operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name', dag=self.dag, task_id="task")
|
||||
|
||||
self.task_instance = TaskInstance(task=self.operator, run_id=self.dag_run.run_id)
|
||||
self.task_instance.dag_run = self.dag_run # Associate TaskInstance with DagRun
|
||||
self.task_instance.state = State.RUNNING
|
||||
|
||||
# Set the context to include TaskInstance
|
||||
self.context = {'ti': self.task_instance}
|
||||
|
||||
@mock.patch.object(deployment_status_operator.XcomPuller, 'get_concurrency_status', return_value=True)
|
||||
@mock.patch('shipyard_airflow.conf.config.parse_args')
|
||||
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator,
|
||||
'_get_version_doc')
|
||||
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator,
|
||||
'_get_status_and_revision')
|
||||
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator,
|
||||
'_store_as_config_map')
|
||||
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_get_version_doc')
|
||||
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_get_status_and_revision')
|
||||
@mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_store_as_config_map')
|
||||
def test_execute(self,
|
||||
store_as_config_map,
|
||||
get_status_and_revision,
|
||||
get_version_doc,
|
||||
config_parse_args,
|
||||
xcom_puller,
|
||||
base_operator_init):
|
||||
get_concurrency_status_mock):
|
||||
|
||||
# Mock return values for the methods
|
||||
get_status_and_revision.return_value = (self.status, self.revision_id)
|
||||
get_version_doc.return_value = self.version
|
||||
|
||||
operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name')
|
||||
|
||||
operator.execute(self.context)
|
||||
# Execute the operator
|
||||
self.operator.execute(self.context)
|
||||
|
||||
assert config_parse_args.called
|
||||
assert xcom_puller.called
|
||||
assert get_concurrency_status_mock.called
|
||||
assert get_status_and_revision.called
|
||||
get_version_doc.assert_called_once_with(self.revision_id)
|
||||
store_as_config_map.assert_called_once_with(self.config_map_data)
|
||||
@ -78,12 +100,10 @@ class TestDeploymentStatusOperator(TestCase):
|
||||
@mock.patch('shipyard_airflow.conf.config.parse_args')
|
||||
def test_execute_concurrency_fail(self,
|
||||
config_parse_args,
|
||||
xcom_puller,
|
||||
base_operator_init):
|
||||
operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name')
|
||||
xcom_puller):
|
||||
|
||||
try:
|
||||
operator.execute(self.context)
|
||||
self.operator.execute(self.context)
|
||||
except airflow.AirflowException as err:
|
||||
assert str(err) == "Concurrency check did not pass, so the " \
|
||||
"deployment status will not be updated"
|
||||
@ -104,15 +124,13 @@ class TestDeploymentStatusOperator(TestCase):
|
||||
create_cfg_map_object,
|
||||
create_cfg_map,
|
||||
patch_cfg_map,
|
||||
load_k8s_config,
|
||||
base_operator_init):
|
||||
load_k8s_config):
|
||||
data = 'data'
|
||||
|
||||
patch_cfg_map.side_effect = kubernetes.client.rest.ApiException(
|
||||
status=404)
|
||||
operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name')
|
||||
operator._store_as_config_map(data)
|
||||
|
||||
self.operator._store_as_config_map(data)
|
||||
patch_cfg_map.assert_called_once_with(
|
||||
config_parser.return_value,
|
||||
config_parser.return_value,
|
||||
@ -144,13 +162,11 @@ class TestDeploymentStatusOperator(TestCase):
|
||||
create_cfg_map,
|
||||
patch_cfg_map,
|
||||
load_k8s_config,
|
||||
get_deployment_status,
|
||||
base_operator_init):
|
||||
get_deployment_status):
|
||||
data = 'data'
|
||||
|
||||
operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name')
|
||||
operator._store_as_config_map(data)
|
||||
|
||||
self.operator._store_as_config_map(data)
|
||||
patch_cfg_map.assert_called_once_with(
|
||||
config_parser.return_value,
|
||||
config_parser.return_value,
|
||||
@ -173,16 +189,14 @@ class TestDeploymentStatusOperator(TestCase):
|
||||
config_parser,
|
||||
create_cfg_map_object,
|
||||
patch_cfg_map,
|
||||
load_k8s_config,
|
||||
base_operator_init):
|
||||
load_k8s_config):
|
||||
data = 'data'
|
||||
|
||||
patch_cfg_map.side_effect = kubernetes.client.rest.ApiException(
|
||||
status=409)
|
||||
operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name')
|
||||
|
||||
try:
|
||||
operator._store_as_config_map(data)
|
||||
self.operator._store_as_config_map(data)
|
||||
except kubernetes.client.rest.ApiException as err:
|
||||
assert patch_cfg_map.side_effect == err
|
||||
|
||||
@ -206,11 +220,9 @@ class TestDeploymentStatusOperator(TestCase):
|
||||
def test__get_version_doc(self,
|
||||
config_parser,
|
||||
get_unique_doc,
|
||||
get_client,
|
||||
base_operator_init):
|
||||
operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name')
|
||||
result = operator._get_version_doc(self.revision_id)
|
||||
get_client):
|
||||
|
||||
result = self.operator._get_version_doc(self.revision_id)
|
||||
|
||||
assert result == get_unique_doc.return_value
|
||||
assert get_client.called
|
||||
@ -229,13 +241,11 @@ class TestDeploymentStatusOperator(TestCase):
|
||||
def test__get_version_doc_does_not_exist(self,
|
||||
config_parser,
|
||||
get_unique_doc,
|
||||
get_client,
|
||||
base_operator_init):
|
||||
get_client):
|
||||
get_unique_doc.side_effect = errors.DocumentNotFoundError()
|
||||
|
||||
operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name')
|
||||
result = operator._get_version_doc(self.revision_id)
|
||||
|
||||
result = self.operator._get_version_doc(self.revision_id)
|
||||
|
||||
assert result == {}
|
||||
assert get_client.called
|
||||
@ -247,21 +257,18 @@ class TestDeploymentStatusOperator(TestCase):
|
||||
@mock.patch('shipyard_airflow.plugins.deployment_status_operator'
|
||||
'.get_deployment_status')
|
||||
def test__get_status_and_revision(self,
|
||||
get_deployment_status,
|
||||
base_operator_init):
|
||||
operator = deployment_status_operator.DeploymentStatusOperator(
|
||||
shipyard_conf='conf', main_dag_name='name')
|
||||
get_deployment_status):
|
||||
|
||||
action = {'committed_rev_id': self.revision_id}
|
||||
operator.xcom_puller = mock.MagicMock()
|
||||
operator.xcom_puller.get_action_info = mock.MagicMock()
|
||||
operator.xcom_puller.get_action_info.return_value = action
|
||||
self.operator.xcom_puller = mock.MagicMock()
|
||||
self.operator.xcom_puller.get_action_info = mock.MagicMock()
|
||||
self.operator.xcom_puller.get_action_info.return_value = action
|
||||
|
||||
status_and_revision = operator._get_status_and_revision()
|
||||
status_and_revision = self.operator._get_status_and_revision()
|
||||
|
||||
get_deployment_status.assert_called_once_with(
|
||||
action,
|
||||
force_completed=operator.force_completed)
|
||||
force_completed=self.operator.force_completed)
|
||||
|
||||
assert status_and_revision == (get_deployment_status.return_value,
|
||||
self.revision_id)
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[tox]
|
||||
skipsdist=True
|
||||
envlist = py38, pep8, bandit
|
||||
envlist = py310, pep8, bandit
|
||||
|
||||
[testenv]
|
||||
setenv=
|
||||
@ -17,20 +17,30 @@ allowlist_externals=
|
||||
sh
|
||||
deps=
|
||||
-r{toxinidir}/requirements-direct.txt
|
||||
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt
|
||||
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
|
||||
commands=
|
||||
rm -f {toxinidir}/requirements-frozen.txt
|
||||
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
|
||||
|
||||
[testenv:py38]
|
||||
[testenv:safety]
|
||||
basepython = python3
|
||||
deps =
|
||||
safety
|
||||
allowlist_externals=
|
||||
safety
|
||||
commands =
|
||||
safety check -r {toxinidir}/requirements-frozen.txt --full-report
|
||||
|
||||
[testenv:py310]
|
||||
skipsdist=True
|
||||
setenv=
|
||||
SLUGIFY_USES_TEXT_UNIDECODE=yes
|
||||
PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
|
||||
basepython=python3.8
|
||||
basepython=python3.10
|
||||
allowlist_externals=
|
||||
bash
|
||||
airflow
|
||||
../../../tools/wait_for_dag.sh
|
||||
commands =
|
||||
bash -c "rm -rf $HOME/airflow"
|
||||
airflow version
|
||||
@ -39,9 +49,14 @@ commands =
|
||||
airflow info
|
||||
airflow dags list
|
||||
airflow dags list-import-errors
|
||||
bash -c "nohup airflow scheduler >/dev/null 2>&1 &"
|
||||
../../../tools/wait_for_dag.sh
|
||||
airflow dags list
|
||||
airflow dags unpause example_bash_operator
|
||||
airflow tasks test example_bash_operator runme_0
|
||||
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
|
||||
airflow tasks run example_bash_operator runme_0 2018-01-01
|
||||
airflow tasks states-for-dag-run example_bash_operator backfill__2018-01-01T00:00:00+00:00
|
||||
airflow dags state example_bash_operator 2018-01-01
|
||||
pytest {posargs} -vv
|
||||
|
||||
@ -53,6 +68,7 @@ setenv=
|
||||
allowlist_externals=
|
||||
bash
|
||||
airflow
|
||||
../../../tools/wait_for_dag.sh
|
||||
commands =
|
||||
bash -c "rm -rf $HOME/airflow"
|
||||
airflow version
|
||||
@ -61,9 +77,15 @@ commands =
|
||||
airflow info
|
||||
airflow dags list
|
||||
airflow dags list-import-errors
|
||||
bash -c "nohup airflow scheduler >/dev/null 2>&1 &"
|
||||
../../../tools/wait_for_dag.sh
|
||||
airflow dags list
|
||||
airflow dags unpause example_bash_operator
|
||||
airflow tasks test example_bash_operator runme_0
|
||||
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
|
||||
airflow tasks run example_bash_operator runme_0 2018-01-01
|
||||
airflow tasks states-for-dag-run example_bash_operator backfill__2018-01-02T00:00:00+00:00
|
||||
airflow dags state example_bash_operator 2018-01-01
|
||||
pytest \
|
||||
{posargs} \
|
||||
--cov-branch \
|
||||
|
||||
@ -1,43 +1,41 @@
|
||||
arrow==1.3.0
|
||||
certifi==2024.2.2
|
||||
certifi==2024.8.30
|
||||
charset-normalizer==3.3.2
|
||||
click==8.1.7
|
||||
click-default-group==1.2.4
|
||||
debtcollector==3.0.0
|
||||
Flask==2.2.5
|
||||
idna==3.6
|
||||
importlib-metadata==6.11.0
|
||||
idna==3.10
|
||||
iso8601==2.1.0
|
||||
itsdangerous==2.1.2
|
||||
Jinja2==3.1.3
|
||||
itsdangerous==2.2.0
|
||||
Jinja2==3.1.4
|
||||
keystoneauth1==5.1.2
|
||||
MarkupSafe==2.1.5
|
||||
msgpack==1.0.7
|
||||
netaddr==1.2.1
|
||||
msgpack==1.1.0
|
||||
netaddr==1.3.0
|
||||
netifaces==0.11.0
|
||||
os-service-types==1.7.0
|
||||
oslo.config==9.1.1
|
||||
oslo.i18n==6.0.0
|
||||
oslo.serialization==5.1.1
|
||||
oslo.utils==6.1.0
|
||||
packaging==23.2
|
||||
pbr==6.0.0
|
||||
pip==23.2.1
|
||||
packaging==24.1
|
||||
pbr==6.1.0
|
||||
pip==24.1
|
||||
PTable==0.9.2
|
||||
pylibyaml==0.1.0
|
||||
pyparsing==3.1.1
|
||||
python-dateutil==2.8.2
|
||||
pyparsing==3.1.4
|
||||
python-dateutil==2.9.0.post0
|
||||
python-keystoneclient==5.1.0
|
||||
pytz==2024.1
|
||||
PyYAML==6.0.1
|
||||
requests==2.31.0
|
||||
rfc3986==1.5.0
|
||||
setuptools==68.2.2
|
||||
pytz==2024.2
|
||||
PyYAML==6.0.2
|
||||
requests==2.32.3
|
||||
rfc3986==2.0.0
|
||||
setuptools==70.1.0
|
||||
six==1.16.0
|
||||
stevedore==5.2.0
|
||||
types-python-dateutil==2.8.19.20240106
|
||||
urllib3==1.26.18
|
||||
stevedore==5.3.0
|
||||
types-python-dateutil==2.9.0.20240906
|
||||
urllib3==2.2.2
|
||||
Werkzeug==2.2.3
|
||||
wheel==0.41.2
|
||||
wheel==0.43.0
|
||||
wrapt==1.16.0
|
||||
zipp==3.17.0
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[tox]
|
||||
skipsdist=True
|
||||
envlist = py38, pep8, bandit
|
||||
envlist = py310, pep8, bandit
|
||||
|
||||
[testenv]
|
||||
setenv=
|
||||
@ -18,12 +18,22 @@ allowlist_externals=
|
||||
sh
|
||||
deps=
|
||||
-r{toxinidir}/requirements-direct.txt
|
||||
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt
|
||||
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
|
||||
commands=
|
||||
rm -f {toxinidir}/requirements-frozen.txt
|
||||
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
|
||||
|
||||
[testenv:py38]
|
||||
[testenv:safety]
|
||||
basepython = python3
|
||||
deps =
|
||||
safety
|
||||
allowlist_externals=
|
||||
safety
|
||||
commands =
|
||||
safety check -r {toxinidir}/requirements-frozen.txt --full-report
|
||||
|
||||
|
||||
[testenv:py310]
|
||||
skipsdist=True
|
||||
commands =
|
||||
pytest \
|
||||
|
||||
@ -32,6 +32,7 @@ fi
|
||||
if [ "${USE_PROXY}" == "true" ]; then
|
||||
TEST_RESP="$(docker run \
|
||||
-p 8080:8080 \
|
||||
--rm \
|
||||
--env HTTP_PROXY="${PROXY}" \
|
||||
--env HTTPS_PROXY="${PROXY}" \
|
||||
--env NO_PROXY="${NO_PROXY}" \
|
||||
@ -40,12 +41,11 @@ if [ "${USE_PROXY}" == "true" ]; then
|
||||
else
|
||||
TEST_RESP="$(docker run \
|
||||
-p 8080:8080 \
|
||||
--rm \
|
||||
--name airflow_test ${IMAGE} \
|
||||
quicktest)"
|
||||
fi
|
||||
|
||||
docker stop airflow_test
|
||||
docker rm airflow_test
|
||||
|
||||
|
||||
if [ ${TEST_RESP:(-7)} == "success" ]; then
|
||||
|
||||
31
tools/gate/playbooks/checkout-treasuremap-ref.yaml
Normal file
31
tools/gate/playbooks/checkout-treasuremap-ref.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
---
|
||||
- hosts: all
|
||||
|
||||
tasks:
|
||||
- name: Checkout treasuremap ref
|
||||
shell: |
|
||||
set -xe;
|
||||
|
||||
: "${TREASUREMAP_REF:=v1.9}"
|
||||
|
||||
cd ../treasuremap
|
||||
git fetch https://review.opendev.org/airship/treasuremap ${TREASUREMAP_REF} && git checkout FETCH_HEAD
|
||||
|
||||
args:
|
||||
chdir: "{{ zuul.project.src_dir }}"
|
||||
environment:
|
||||
TREASUREMAP_REF: "{{ treasuremap_ref }}"
|
||||
|
||||
...
|
||||
@ -15,4 +15,7 @@
|
||||
roles:
|
||||
- bindep
|
||||
- start-zuul-console
|
||||
- role: add-authorized-keys
|
||||
public_keys:
|
||||
- public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
|
||||
...
|
||||
|
||||
@ -19,7 +19,7 @@ osh_params:
|
||||
container_distro_version: focal
|
||||
# feature_gates:
|
||||
site: airskiff
|
||||
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
|
||||
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
|
||||
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
|
||||
OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
|
||||
OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
|
||||
|
||||
@ -35,7 +35,7 @@
|
||||
FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
|
||||
RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
|
||||
PL_SITE: "{{ site | default('airskiff') }}"
|
||||
HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
|
||||
HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz') }}"
|
||||
HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
|
||||
OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
|
||||
OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
|
||||
|
||||
@ -19,7 +19,7 @@ osh_params:
|
||||
container_distro_version: focal
|
||||
# feature_gates:
|
||||
site: airskiff
|
||||
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
|
||||
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz
|
||||
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
|
||||
OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
|
||||
OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
|
||||
|
||||
@ -32,7 +32,7 @@
|
||||
FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
|
||||
RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
|
||||
PL_SITE: "{{ site | default('airskiff') }}"
|
||||
HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
|
||||
HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz') }}"
|
||||
HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
|
||||
OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
|
||||
OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
set -x
|
||||
|
||||
HELM=$1
|
||||
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}
|
||||
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz"}
|
||||
|
||||
|
||||
function install_helm_binary {
|
||||
|
||||
10
tools/wait_for_dag.sh
Executable file
10
tools/wait_for_dag.sh
Executable file
@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
while true; do
|
||||
is_active=$(airflow dags details example_bash_operator -o plain | grep is_active | awk '{print $2}')
|
||||
if [ "$is_active" == "True" ]; then
|
||||
echo "DAG example_bash_operator is active."
|
||||
break
|
||||
fi
|
||||
echo "Waiting for DAG example_bash_operator to become active..."
|
||||
sleep 10
|
||||
done
|
||||
35
tox.ini
35
tox.ini
@ -16,6 +16,27 @@ passenv=
|
||||
[testenv:venv]
|
||||
commands = {posargs}
|
||||
|
||||
[testenv:airflow_safety]
|
||||
basepython = python3
|
||||
deps =
|
||||
safety
|
||||
allowlist_externals=
|
||||
safety
|
||||
tox
|
||||
commands =
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e safety
|
||||
|
||||
[testenv:shipyard_safety]
|
||||
basepython = python3
|
||||
deps =
|
||||
safety
|
||||
allowlist_externals=
|
||||
safety
|
||||
tox
|
||||
commands =
|
||||
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e safety
|
||||
|
||||
|
||||
# Please use `make docs` instead
|
||||
[testenv:docs]
|
||||
allowlist_externals=
|
||||
@ -36,22 +57,22 @@ commands=
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini
|
||||
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini
|
||||
|
||||
[testenv:py38]
|
||||
deps =
|
||||
tox<=4.0.0
|
||||
[testenv:py310]
|
||||
; deps =
|
||||
; tox
|
||||
allowlist_externals=
|
||||
tox
|
||||
bash
|
||||
commands=
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38
|
||||
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py310
|
||||
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py310
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all
|
||||
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
|
||||
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
|
||||
|
||||
[testenv:cover]
|
||||
deps =
|
||||
tox<=4.0.0
|
||||
; deps =
|
||||
; tox
|
||||
allowlist_externals=
|
||||
tox
|
||||
sh
|
||||
|
||||
Loading…
Reference in New Issue
Block a user