diff --git a/.zuul.yaml b/.zuul.yaml index b79c05e6..787b9752 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,30 +13,34 @@ - project: templates: - docs-on-readthedocs - - openstack-python38-jobs vars: rtd_webhook_id: '38576' rtd_project_name: 'airship-shipyard' ensure_global_symlinks: true check: jobs: - - airship-shipyard-openstack-tox-pep8-focal - - airship-shipyard-openstack-tox-cover-focal + - openstack-tox-py310 + - airship-shipyard-openstack-tox-pep8-jammy + - airship-shipyard-openstack-tox-cover-jammy - airship-shipyard-chart-build-gate - airship-shipyard-chart-build-latest-htk + - airship-airflow-dependency-vulnerability-check + - airship-shipyard-dependency-vulnerability-check - airship-shipyard-whitespace-lint-gate - - airship-shipyard-airskiff-deployment-focal - - shipyard-airskiff-deployment-focal-kubeadm + - airship-shipyard-image-gate-ubuntu_jammy + - shipyard-airskiff-deployment-jammy-kubeadm gate: jobs: - - airship-shipyard-openstack-tox-pep8-focal - - airship-shipyard-openstack-tox-cover-focal + - openstack-tox-py310 + - airship-shipyard-openstack-tox-pep8-jammy + - airship-shipyard-openstack-tox-cover-jammy - airship-shipyard-chart-build-gate - airship-shipyard-whitespace-lint-gate - - airship-shipyard-airskiff-deployment-focal + - airship-shipyard-image-gate-ubuntu_jammy + - shipyard-airskiff-deployment-jammy-kubeadm post: jobs: - - airship-shipyard-docker-build-publish-ubuntu_focal + - airship-shipyard-docker-build-publish-ubuntu_jammy - shipyard-upload-git-mirror - nodeset: @@ -46,32 +50,52 @@ label: ubuntu-bionic - nodeset: - name: airship-shipyard-single-node-focal + name: airship-shipyard-single-node-jammy nodes: - name: primary - label: ubuntu-focal + label: ubuntu-jammy - job: - name: airship-shipyard-openstack-tox-pep8-focal + name: airship-airflow-dependency-vulnerability-check + parent: tox-py310 + voting: false + timeout: 600 + nodeset: ubuntu-jammy + vars: + tox_envlist: airflow_safety + bindep_profile: test py310 + +- job: + name: airship-shipyard-dependency-vulnerability-check + parent: tox-py310 + voting: false + timeout: 600 + nodeset: ubuntu-jammy + vars: + tox_envlist: shipyard_safety + bindep_profile: test py310 + +- job: + name: airship-shipyard-openstack-tox-pep8-jammy parent: openstack-tox-pep8 - description: Runs pep8 job on focal - nodeset: airship-shipyard-single-node-focal + description: Runs pep8 job on jammy + nodeset: airship-shipyard-single-node-jammy - job: - name: airship-shipyard-openstack-tox-cover-focal + name: airship-shipyard-openstack-tox-cover-jammy parent: openstack-tox-cover - description: Runs cover job on focal - nodeset: airship-shipyard-single-node-focal + description: Runs cover job on jammy + nodeset: airship-shipyard-single-node-jammy - job: name: airship-shipyard-chart-build-gate description: Build charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: airship-shipyard-single-node-focal + nodeset: airship-shipyard-single-node-jammy vars: - HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f + HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b - job: name: airship-shipyard-chart-build-latest-htk @@ -79,7 +103,7 @@ timeout: 900 voting: false run: tools/gate/playbooks/build-charts.yaml - nodeset: airship-shipyard-single-node-focal + nodeset: airship-shipyard-single-node-jammy vars: HTK_COMMIT: master @@ -88,7 +112,7 @@ description: | Lints all files by checking them for whitespace. run: tools/gate/playbooks/zuul-linter.yaml - nodeset: airship-shipyard-single-node-focal + nodeset: airship-shipyard-single-node-jammy - job: name: airship-shipyard-image-base @@ -96,7 +120,7 @@ Base job for running airship-shipyard image related jobs. timeout: 3600 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: airship-shipyard-single-node-focal + nodeset: airship-shipyard-single-node-jammy irrelevant-files: &irrelevant-files - ^.*\.rst$ - ^doc/.*$ @@ -104,39 +128,17 @@ - job: - name: airship-shipyard-image-gate-ubuntu_focal + name: airship-shipyard-image-gate-ubuntu_jammy description: | - Run shipyard-image build for ubuntu_focal. + Run shipyard-image build for ubuntu_jammy. parent: airship-shipyard-image-base vars: publish: false - distro: ubuntu_focal + distro: ubuntu_jammy tags: dynamic: patch_set: true -# - job: -# name: airship-shipyard-airskiff-deployment-focal -# nodeset: airship-shipyard-single-node-focal -# description: | -# Deploy Memcached using Airskiff and submitted Armada changes. -# timeout: 9600 -# voting: true -# pre-run: -# - tools/gate/playbooks/airskiff-reduce-site.yaml -# - tools/gate/playbooks/git-config.yaml -# run: tools/gate/playbooks/airskiff-deploy.yaml -# post-run: tools/gate/playbooks/debug-report.yaml -# required-projects: -# - airship/treasuremap -# vars: -# CLONE_SHIPYARD: false -# SHIPYARD_IMAGE_DISTRO: ubuntu_focal -# HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz -# HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f -# OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f -# OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7 -# irrelevant-files: *irrelevant-files - job: name: shipyard-base @@ -153,6 +155,7 @@ irrelevant-files: *irrelevant-files timeout: 10800 pre-run: + - tools/gate/playbooks/checkout-treasuremap-ref.yaml - tools/gate/playbooks/prepare-hosts.yaml - tools/gate/playbooks/mount-volumes.yaml - tools/gate/playbooks/deploy-env.yaml @@ -161,6 +164,7 @@ post-run: - tools/gate/playbooks/osh-infra-collect-logs.yaml vars: + treasuremap_ref: refs/changes/43/927643/4 extra_volume: size: 80G type: Linux @@ -177,16 +181,16 @@ loopback_device: /dev/loop100 loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img" ceph_osd_data_device: /dev/loop100 - kube_version_repo: "v1.29" - # the list of k8s package versions are available here - # https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages - kube_version: "1.29.5-1.1" + kube_version_repo: "v1.30" + kube_version: "1.30.3-1.1" calico_setup: true - calico_version: "v3.27.0" + calico_version: "v3.27.4" cilium_setup: false - helm_version: "v3.14.2" - yq_version: "v4.6.0" - crictl_version: "v1.26.1" + cilium_version: "1.16.0" + flannel_setup: false + flannel_version: v0.25.4 + helm_version: "v3.15.4" + crictl_version: "v1.30.1" zuul_osh_relative_path: ../../openstack/openstack-helm zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra zuul_treasuremap_relative_path: ../../airship/treasuremap @@ -194,19 +198,20 @@ run_helm_tests: "no" - job: - name: shipyard-airskiff-deployment-focal-kubeadm - nodeset: treasuremap-airskiff-1node-ubuntu_focal + name: shipyard-airskiff-deployment-jammy-kubeadm + nodeset: treasuremap-airskiff-1node-ubuntu_jammy description: | Deploy Memcached using Airskiff and submitted Shipyard changes. parent: shipyard-base vars: site: airskiff - HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.14.2-linux-amd64.tar.gz - HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8 - OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8 - OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015 + HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz + HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b + OSH_INFRA_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b + OSH_COMMIT: 540df5cb0dbdaed63c202e2d6f2b7891062f8203 + TREASUREMAP_COMMIT: refs/changes/43/927643/4 CLONE_SHIPYARD: false - DISTRO: ubuntu_focal + DISTRO: ubuntu_jammy DOCKER_REGISTRY: localhost:5000 MAKE_SHIPYARD_IMAGES: true gate_scripts_relative_path: ../../airship/treasuremap @@ -227,23 +232,23 @@ irrelevant-files: *irrelevant-files - job: - name: airship-shipyard-docker-build-publish-ubuntu_focal + name: airship-shipyard-docker-build-publish-ubuntu_jammy timeout: 3600 run: tools/gate/playbooks/run-image.yaml - nodeset: airship-shipyard-single-node-focal + nodeset: airship-shipyard-single-node-jammy irrelevant-files: *irrelevant-files secrets: - airship_shipyard_quay_creds vars: publish: true - distro: ubuntu_focal + distro: ubuntu_jammy tags: dynamic: branch: true commit: true static: - latest - - airflow_2.8.2 + - airflow_2.10.2 diff --git a/Makefile b/Makefile index 375a3fc7..572a83c1 100644 --- a/Makefile +++ b/Makefile @@ -34,9 +34,11 @@ USE_PROXY ?= false AIRFLOW_SRC ?= AIRFLOW_HOME ?= DISTRO_BASE_IMAGE ?= -DISTRO ?= ubuntu_focal +DISTRO ?= ubuntu_jammy +DISTRO_ALIAS ?= ubuntu_focal IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO} +IMAGE_ALIAS := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO_ALIAS} IMAGE_DIR:=images/$(IMAGE_NAME) .PHONY: images @@ -137,6 +139,13 @@ else $(_AIRFLOW_HOME_ARG) \ --build-arg ctx_base=$(BUILD_CTX) . endif +ifneq ($(DISTRO), $(DISTRO_ALIAS)) + docker tag $(IMAGE) $(IMAGE_ALIAS) +endif +ifeq ($(DOCKER_REGISTRY), localhost:5000) + docker push $(IMAGE) + docker push $(IMAGE_ALIAS) +endif ifeq ($(PUSH_IMAGE), true) docker push $(IMAGE) endif diff --git a/charts/shipyard/Chart.yaml b/charts/shipyard/Chart.yaml index 13463b3e..13b17f1e 100644 --- a/charts/shipyard/Chart.yaml +++ b/charts/shipyard/Chart.yaml @@ -16,8 +16,8 @@ apiVersion: v1 description: A Helm chart for Shipyard and Airflow name: shipyard -version: 0.2.6 -appVersion: 2.8.2 +version: 0.2.7 +appVersion: 2.10.2 keywords: - shipyard - airflow diff --git a/images/airflow/Dockerfile.ubuntu_focal b/images/airflow/Dockerfile.ubuntu_focal index 3dbe9f08..bd978c67 100644 --- a/images/airflow/Dockerfile.ubuntu_focal +++ b/images/airflow/Dockerfile.ubuntu_focal @@ -180,4 +180,4 @@ USER airflow WORKDIR ${AIRFLOW_HOME} # Execute entrypoint -ENTRYPOINT ["./entrypoint.sh"] +ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/images/airflow/Dockerfile.ubuntu_jammy b/images/airflow/Dockerfile.ubuntu_jammy new file mode 100644 index 00000000..641ffc21 --- /dev/null +++ b/images/airflow/Dockerfile.ubuntu_jammy @@ -0,0 +1,186 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Docker image to run Airflow on Kubernetes +ARG FROM=ubuntu:jammy +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \ + org.opencontainers.image.source='https://opendev.org/airship/shipyard' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +# Do not prompt user for choices on installation/configuration of packages +# Set port 8080 for Airflow Web +# Set port 5555 for Airflow Flower +# Set port 8793 for Airflow Worker +ENV container docker +ENV WEB_PORT 8080 +ENV FLOWER_PORT 5555 +ENV WORKER_PORT 8793 +ENV SLUGIFY_USES_TEXT_UNIDECODE yes +ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning + +# Expose port for applications +EXPOSE $WEB_PORT +EXPOSE $FLOWER_PORT +EXPOSE $WORKER_PORT + +# Set ARG for usage during build +ARG AIRFLOW_HOME=/usr/local/airflow +# Moved celery to images/airflow/requirements.txt as apache-airflow uses a +# version of celery incompatibile with the version of kombu needed by other +# Airship components +ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2" +ARG DEBIAN_FRONTEND=noninteractive +ARG ctx_base=src/bin + +# Kubectl version +ARG KUBECTL_VERSION=1.31.0 + +# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated +ENV AIRFLOW_HOME=${AIRFLOW_HOME} + +RUN set -ex \ + && apt-get update && apt-get upgrade -y \ + && apt-get install -y wget curl \ + apt-transport-https ca-certificates gnupg \ + && echo "deb http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && curl -o /etc/apt/trusted.gpg.d/postgresql_release_signing_key.asc 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' \ + && apt update \ + && apt -y install \ + automake \ + build-essential \ + ca-certificates \ + curl \ + git \ + g++ \ + libkrb5-dev \ + libffi-dev \ + libssl-dev \ + libpq-dev \ + libpq5 \ + libtool \ + libxml2 \ + libsasl2-dev \ + locales \ + netcat \ + netbase \ + postgresql-client \ + postgresql-common \ + python3 \ + python3-setuptools \ + python3-pip \ + python3-dev \ + python3-dateutil \ + make \ + --no-install-recommends \ + && python3 -m pip install -U pip \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/doc-base \ + /etc/ssl/private/ssl-cert-snakeoil.key \ + /etc/ssl/private/ssl-cert-snakeoil.pem + +# Install LibYAML +ENV LD_LIBRARY_PATH=/usr/local/lib + +ARG LIBYAML_VERSION=0.2.5 +RUN set -ex \ + && git clone https://github.com/yaml/libyaml.git \ + && cd libyaml \ + && git checkout $LIBYAML_VERSION \ + && ./bootstrap \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -fr libyaml + + +# Things that change mostly infrequently +RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \ + && curl -L -o /usr/local/bin/kubectl \ + https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ + && chmod +x /usr/local/bin/kubectl + +# Dependency requirements +# Note - removing snakebite (python 2 vs. 3). See: +# https://github.com/puckel/docker-airflow/issues/77 +# Install Airflow directly to allow overriding source +# COPY images/airflow/requirements.txt /tmp/ +COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt +RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \ + && pip3 install $AIRFLOW_SRC --no-cache-dir + # && (pip3 uninstall -y snakebite || true) \ + # && (pip3 uninstall -y psycopg2 || true) \ + # && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true) + +# Copy scripts used in the container: +COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/ + +# Copy configuration (e.g. logging config for Airflow): +COPY images/airflow/config/*.py ${AIRFLOW_HOME}/config/ +COPY images/airflow/webserver_config.py ${AIRFLOW_HOME}/ + +# Change permissions +RUN chown -R airflow: ${AIRFLOW_HOME} + +# Setting the version explicitly for PBR +ENV PBR_VERSION 0.1a1 + +# Shipyard +# +# Shipyard provides core functionality used by the Airflow plugins/operators +# Since Shipyard and Airflow are built together as images, this should prevent +# stale or out-of-date code between these parts. +# Shipyard requirements, source and installation +# COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt +# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir + +COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/ +RUN cd /tmp/shipyard \ + && pip3 install $(pwd) --use-pep517 + +# Note: The value for the dags and plugins directories that are sourced +# from the values.yaml of the Shipyard Helm chart need to align with these +# directories. If they do not, airflow will not find the intended dags and +# plugins. +# +# Note: In the case of building images using the provided Makefile, a test is +# run against the built-in dags provided with Airflow. Since there is no Helm +# chart to reconfigure the airflow.cfg with these directories, these dags and +# plugins are not known to Airflow during the image test. +# +# Copy the plugins and dags that will be used by this Airflow image: +COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/ +COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins \ + /usr/local/lib/python3.10/dist-packages/airflow/plugins/ +COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/ +COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags \ + /usr/local/lib/python3.10/dist-packages/airflow/dags/ + +# Set work directory +USER airflow +WORKDIR ${AIRFLOW_HOME} + +# Execute entrypoint +ENTRYPOINT ["./entrypoint.sh"] diff --git a/images/airflow/script/airflow_start_service.sh b/images/airflow/script/airflow_start_service.sh index 82dc2865..eaf62b14 100755 --- a/images/airflow/script/airflow_start_service.sh +++ b/images/airflow/script/airflow_start_service.sh @@ -40,13 +40,26 @@ elif [[ $cmd == *scheduler* ]]; then ${python3_path} ${airflow_path} scheduler $2 $3 done elif [[ $cmd == 'quicktest' ]]; then + ${python3_path} ${airflow_path} version ${python3_path} ${airflow_path} db init ${python3_path} ${airflow_path} db migrate ${python3_path} ${airflow_path} dags list - ${python3_path} ${airflow_path} webserver -p 8080 & + ${python3_path} ${airflow_path} scheduler & + while true; do + is_active=$(airflow dags details example_bash_operator -o plain | grep is_active | awk '{print $2}') + if [ "$is_active" == "True" ]; then + echo "DAG example_bash_operator is active." + break + fi + echo "Waiting for DAG example_bash_operator to become active..." + sleep 10 + done + ${python3_path} ${airflow_path} dags list + ${python3_path} ${airflow_path} dags unpause example_bash_operator ${python3_path} ${airflow_path} tasks test example_bash_operator runme_0 ${python3_path} ${airflow_path} dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 ${python3_path} ${airflow_path} tasks run example_bash_operator runme_0 2018-01-01 + ${python3_path} ${airflow_path} tasks states-for-dag-run example_bash_operator backfill__2018-01-02T00:00:00+00:00 ${python3_path} ${airflow_path} dags state example_bash_operator 2018-01-01 else echo "Invalid Command!" diff --git a/images/shipyard/Dockerfile.ubuntu_jammy b/images/shipyard/Dockerfile.ubuntu_jammy new file mode 100644 index 00000000..99d0e524 --- /dev/null +++ b/images/shipyard/Dockerfile.ubuntu_jammy @@ -0,0 +1,141 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG FROM=ubuntu:jammy +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://airship-shipyard.readthedocs.org' \ + org.opencontainers.image.source='https://opendev.org/airship/shipyard' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +ENV container docker +ENV PORT 9000 +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning + +# Setting the version explicitly for PBR +ENV PBR_VERSION 0.1a1 + +ARG DEBIAN_FRONTEND=noninteractive +ARG ctx_base=src/bin + +# Expose port 9000 for application +EXPOSE $PORT + +RUN set -ex && \ + apt-get update && apt-get upgrade -y && \ + apt-get -y install \ + automake \ + ca-certificates \ + curl \ + build-essential \ + git \ + libtool \ + libxml2 \ + make \ + netbase \ + python3-dev \ + python3-setuptools \ + --no-install-recommends \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/doc-base + +# Install LibYAML +ENV LD_LIBRARY_PATH=/usr/local/lib + +ARG LIBYAML_VERSION=0.2.5 +RUN set -ex \ + && git clone https://github.com/yaml/libyaml.git \ + && cd libyaml \ + && git checkout $LIBYAML_VERSION \ + && ./bootstrap \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -fr libyaml + +# Create shipyard user +RUN useradd -ms /bin/bash shipyard \ + && mkdir -p /home/shipyard/shipyard \ + && mkdir -p /home/shipyard/shipyard_client + +# Copy entrypoint.sh to /home/shipyard +COPY ${ctx_base}/shipyard_airflow/entrypoint.sh /home/shipyard/entrypoint.sh +# Change permissions and set up directories +RUN chown -R shipyard: /home/shipyard \ + && chmod +x /home/shipyard/entrypoint.sh + +# Requirements and Shipyard source +COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /home/shipyard/api_requirements.txt +COPY ${ctx_base}/shipyard_client/requirements-frozen.txt /home/shipyard/client_requirements.txt +COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/ +COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/ + +# Build + RUN set -ex \ + && buildDeps=' \ + build-essential \ + git \ + libssl-dev \ + libffi-dev \ + libkrb5-dev \ + libpq-dev \ + libsasl2-dev \ + libssl-dev \ + libre2-dev \ + libzmq3-dev \ + make \ + python3-pip \ + ' \ + && apt-get -qq update \ + && apt-get upgrade -y \ + && apt-get -y install -y $buildDeps --no-install-recommends \ + && python3 -m pip install -U pip \ + && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \ + && cd /home/shipyard/shipyard_client \ + && pip3 install . --use-pep517 --no-cache-dir \ + && pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \ + && cd /home/shipyard/shipyard \ + && pip3 install . --use-pep517 --no-cache-dir \ + && apt-get purge -y --auto-remove $buildDeps \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/doc-base \ + && python3 -m pip install -U pip + + +# Entrypoint +ENTRYPOINT ["/home/shipyard/entrypoint.sh"] +CMD ["server"] +# Set user to shipyard +USER shipyard diff --git a/src/bin/shipyard_airflow/requirements-direct.txt b/src/bin/shipyard_airflow/requirements-direct.txt index 67bb0d53..da8db90e 100644 --- a/src/bin/shipyard_airflow/requirements-direct.txt +++ b/src/bin/shipyard_airflow/requirements-direct.txt @@ -1,5 +1,5 @@ alembic -apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.8.2 +apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2 arrow celery Flask @@ -59,7 +59,7 @@ oslo.versionedobjects==3.1.0 # Airship dependencies -git+https://opendev.org/airship/deckhand.git@32e9950db405b5d1eb74da0c4e8df344a8638eff#egg=deckhand -git+https://opendev.org/airship/drydock.git@6b317dda0518e3366c9835ee99d9691ffba718eb#egg=drydock_provisioner&subdirectory=python -git+https://opendev.org/airship/armada.git@e85f0ccdeb93475aef2f2f4ce3da9f53a00309cd#egg=armada -git+https://opendev.org/airship/promenade.git@d21e3434f641e1f1e7400a44e77fccb5b87a2824#egg=promenade +git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6#egg=deckhand +git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#egg=drydock_provisioner&subdirectory=python +git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211#egg=armada +git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66#egg=promenade diff --git a/src/bin/shipyard_airflow/requirements-frozen.txt b/src/bin/shipyard_airflow/requirements-frozen.txt index b4fc34a5..7b6cb32f 100644 --- a/src/bin/shipyard_airflow/requirements-frozen.txt +++ b/src/bin/shipyard_airflow/requirements-frozen.txt @@ -1,158 +1,164 @@ -aiohttp==3.9.3 +aiohappyeyeballs==2.4.0 +aiohttp==3.10.5 aiosignal==1.3.1 -alembic==1.13.1 +alembic==1.13.2 amqp==5.2.0 -anyio==4.3.0 -apache-airflow==2.8.2 -apache-airflow-providers-apache-hdfs==4.3.2 -apache-airflow-providers-apache-hive==7.0.0 -apache-airflow-providers-celery==3.6.0 -apache-airflow-providers-common-io==1.3.0 -apache-airflow-providers-common-sql==1.11.0 -apache-airflow-providers-ftp==3.7.0 -apache-airflow-providers-http==4.9.1 -apache-airflow-providers-imap==3.5.0 -apache-airflow-providers-jdbc==4.2.2 -apache-airflow-providers-postgres==5.10.1 -apache-airflow-providers-sqlite==3.7.1 -apispec==6.4.0 -argcomplete==3.2.2 -Armada @ git+https://opendev.org/airship/armada.git@6f06dc09606e94bd6e7322a6d71c1ce5fdd33b0b +anyio==4.4.0 +apache-airflow==2.10.2 +apache-airflow-providers-apache-hdfs==4.5.0 +apache-airflow-providers-apache-hive==8.2.0 +apache-airflow-providers-celery==3.8.1 +apache-airflow-providers-common-compat==1.2.0 +apache-airflow-providers-common-io==1.4.0 +apache-airflow-providers-common-sql==1.16.0 +apache-airflow-providers-fab==1.3.0 +apache-airflow-providers-ftp==3.11.0 +apache-airflow-providers-http==4.13.0 +apache-airflow-providers-imap==3.7.0 +apache-airflow-providers-jdbc==4.5.0 +apache-airflow-providers-postgres==5.12.0 +apache-airflow-providers-smtp==1.8.0 +apache-airflow-providers-sqlite==3.9.0 +apispec==6.6.1 +argcomplete==3.5.0 +Armada @ git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211 arrow==1.3.0 -asgiref==3.7.2 +asgiref==3.8.1 async-timeout==4.0.3 -attrs==23.2.0 +attrs==24.2.0 autopage==0.5.2 -Babel==2.14.0 -backports.zoneinfo==0.2.1 +babel==2.16.0 barbican==16.0.0 -bcrypt==4.1.2 -Beaker==1.12.1 +bcrypt==4.2.0 +Beaker==1.13.0 billiard==4.2.0 -blinker==1.7.0 +blinker==1.8.2 cachelib==0.9.0 -cachetools==5.3.2 -castellan==4.4.0 -celery==5.3.6 -certifi==2024.2.2 -cffi==1.16.0 +cachetools==5.5.0 +castellan==5.1.1 +celery==5.4.0 +certifi==2024.8.30 +cffi==1.17.1 charset-normalizer==3.3.2 click==8.1.7 -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 click-plugins==1.1.1 click-repl==0.3.0 clickclick==20.10.2 -cliff==4.6.0 +cliff==4.7.0 cmd2==2.4.3 colorama==0.4.6 -colorlog==4.8.0 +colorlog==6.8.2 ConfigUpdater==3.2 connexion==2.14.2 -cron-descriptor==1.4.3 -croniter==2.0.1 -cryptography==41.0.7 +cron-descriptor==1.4.5 +croniter==3.0.3 +cryptography==42.0.8 debtcollector==3.0.0 -Deckhand @ git+https://opendev.org/airship/deckhand.git@fd58230f6e31ede4925bab0325105eb5b05ad1a8 +Deckhand @ git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6 decorator==5.1.1 -deepdiff==6.7.1 +deepdiff==8.0.1 Deprecated==1.2.14 dill==0.3.1.1 dnspython==2.6.1 docopt==0.6.2 -docutils==0.20.1 -dogpile.cache==1.3.2 -drydock-provisioner @ git+https://opendev.org/airship/drydock.git@816a4bbe366e2c1d58530986df05f99d7d9a7beb#subdirectory=python -email-validator==1.3.1 -eventlet==0.35.2 -exceptiongroup==1.2.0 +docutils==0.16 +dogpile.cache==1.3.3 +drydock_provisioner @ git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#subdirectory=python +email_validator==2.2.0 +eventlet==0.37.0 +exceptiongroup==1.2.2 falcon==3.1.3 -fastavro==1.9.4 +fastavro==1.9.7 fasteners==0.19 fixtures==4.1.0 Flask==2.2.5 -Flask-AppBuilder==4.3.11 +Flask-AppBuilder==4.5.0 Flask-Babel==2.0.0 -Flask-Caching==2.1.0 +Flask-Caching==2.3.0 Flask-JWT-Extended==4.6.0 -Flask-Limiter==3.5.1 +Flask-Limiter==3.8.0 Flask-Login==0.6.3 Flask-Session==0.5.0 Flask-SQLAlchemy==2.5.1 Flask-WTF==1.2.1 flower==2.0.1 frozenlist==1.4.1 -fsspec==2024.2.0 +fsspec==2024.9.0 future==1.0.0 futurist==3.0.0 gitdb==4.0.11 -GitPython==3.1.42 -google-auth==2.28.1 -google-re2==1.1 -googleapis-common-protos==1.62.0 -graphviz==0.20.1 -greenlet==3.0.3 -grpcio==1.62.0 +GitPython==3.1.43 +google-auth==2.34.0 +google-re2==1.1.20240702 +googleapis-common-protos==1.65.0 +graphviz==0.20.3 +greenlet==3.1.0 +grpcio==1.66.1 gssapi==1.8.3 -gunicorn==21.2.0 +gunicorn==23.0.0 h11==0.14.0 hdfs==2.7.3 hmsclient==0.1.1 html5lib==0.9999999 -httpcore==0.16.3 +httpcore==1.0.5 httpexceptor==1.4.0 -httpx==0.23.3 -humanize==4.9.0 -idna==3.6 +httpx==0.27.0 +humanize==4.10.0 +idna==3.10 importlib-metadata==6.11.0 -importlib-resources==5.13.0 +importlib_resources==6.4.5 inflection==0.5.1 iso8601==2.1.0 -itsdangerous==2.1.2 +itsdangerous==2.2.0 JayDeBeApi==1.2.3 -Jinja2==3.1.3 +Jinja2==3.1.4 +jmespath==0.10.0 JPype1==1.5.0 jsonpath-ng==1.6.1 -jsonpickle==3.0.3 -jsonschema==4.21.1 +jsonpickle==3.3.0 +jsonschema==4.23.0 jsonschema-specifications==2023.12.1 keystoneauth1==5.1.2 keystonemiddleware==10.2.0 -kombu==5.3.5 -krb5==0.5.1 -kubernetes==29.0.0 +kombu==5.4.1 +krb5==0.6.0 +kubernetes==30.1.0 lazy-object-proxy==1.10.0 ldap3==2.9.1 -limits==3.9.0 +limits==3.13.0 linkify-it-py==2.0.3 lockfile==0.12.2 logutils==0.3.5 -Mako==1.3.2 +Mako==1.3.5 markdown-it-py==3.0.0 MarkupSafe==2.1.5 -marshmallow==3.20.2 +marshmallow==3.22.0 marshmallow-oneofschema==3.1.1 -marshmallow-sqlalchemy==0.26.1 -mdit-py-plugins==0.4.0 +marshmallow-sqlalchemy==0.28.2 +mdit-py-plugins==0.4.2 mdurl==0.1.2 -microversion-parse==1.0.1 +methodtools==0.4.7 +microversion-parse==2.0.0 mock==5.1.0 -more-itertools==10.2.0 -msgpack==1.0.7 -multidict==6.0.5 -netaddr==1.2.1 +more-itertools==10.5.0 +msgpack==1.1.0 +multidict==6.1.0 +netaddr==1.3.0 netifaces==0.11.0 -networkx==3.1 -numpy==1.24.4 +networkx==3.3 +numpy==1.26.4 oauthlib==3.2.2 -opentelemetry-api==1.23.0 -opentelemetry-exporter-otlp==1.23.0 -opentelemetry-exporter-otlp-proto-common==1.23.0 -opentelemetry-exporter-otlp-proto-grpc==1.23.0 -opentelemetry-exporter-otlp-proto-http==1.23.0 -opentelemetry-proto==1.23.0 -opentelemetry-sdk==1.23.0 -opentelemetry-semantic-conventions==0.44b0 +opentelemetry-api==1.27.0 +opentelemetry-exporter-otlp==1.27.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 +opentelemetry-proto==1.27.0 +opentelemetry-sdk==1.27.0 +opentelemetry-semantic-conventions==0.48b0 ordered-set==4.1.0 +orderly-set==5.2.2 os-service-types==1.7.0 oslo.cache==3.3.1 oslo.concurrency==5.1.1 @@ -170,115 +176,116 @@ oslo.service==3.1.1 oslo.upgradecheck==2.1.1 oslo.utils==6.1.0 oslo.versionedobjects==3.1.0 -packaging==23.2 -pandas==2.0.3 -Paste==3.7.1 +packaging==24.1 +pandas==2.1.4 +Paste==3.10.1 PasteDeploy==3.1.0 -PasteScript==3.4.0 +PasteScript==3.6.0 pathspec==0.12.1 -pbr==6.0.0 +pbr==6.1.0 pecan==1.5.1 pendulum==3.0.0 -pip==23.2.1 -pkgutil_resolve_name==1.3.10 -pluggy==1.4.0 +pip==24.1 +pluggy==1.5.0 ply==3.11 -prettytable==3.10.0 +prettytable==3.11.0 prison==0.2.1 -promenade @ git+https://opendev.org/airship/promenade.git@14375e658bc06328042b1dc934eefcebf60210fd +promenade @ git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66 prometheus_client==0.20.0 -prompt-toolkit==3.0.43 -protobuf==4.25.3 -psutil==5.9.8 +prompt_toolkit==3.0.47 +protobuf==4.25.4 +psutil==6.0.0 psycopg2-binary==2.9.9 pure-sasl==0.6.2 py==1.11.0 -pyarrow==14.0.2 -pyasn1==0.5.1 -pyasn1-modules==0.3.0 +pyarrow==16.1.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.0 pycadf==3.1.1 -pycparser==2.21 -Pygments==2.17.2 +pycparser==2.22 +Pygments==2.18.0 PyHive==0.7.0 -PyJWT==2.8.0 +PyJWT==2.9.0 pylibyaml==0.1.0 -pyOpenSSL==24.0.0 -pyparsing==3.1.1 -pyperclip==1.8.2 -pyspnego==0.10.2 +pyOpenSSL==24.2.1 +pyparsing==3.1.4 +pyperclip==1.9.0 +pyspnego==0.11.1 python-barbicanclient==5.5.0 python-daemon==3.0.1 -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 python-keystoneclient==5.1.0 python-memcached==1.62 -python-mimeparse==1.6.0 -python-nvd3==0.15.0 +python-mimeparse==2.0.0 +python-nvd3==0.16.0 python-slugify==8.0.4 python3-memcached==1.51 -pytz==2024.1 -PyYAML==6.0.1 -referencing==0.33.0 -regex==2023.12.25 +pytz==2024.2 +PyYAML==6.0.2 +redis==5.0.8 +referencing==0.35.1 +regex==2024.9.11 repoze.lru==0.7 -requests==2.31.0 -requests-kerberos==0.14.0 +requests==2.32.3 +requests-kerberos==0.15.0 requests-oauthlib==1.3.1 requests-toolbelt==1.0.0 resolver==0.2.1 -responses==0.25.0 +responses==0.25.3 retry==0.9.2 rfc3339-validator==0.1.4 -rfc3986==1.5.0 -rich==13.7.0 -rich-argparse==1.4.0 +rfc3986==2.0.0 +rich==13.8.1 +rich-argparse==1.5.2 Routes==2.5.1 -rpds-py==0.18.0 +rpds-py==0.20.0 rsa==4.9 selector==0.10.1 setproctitle==1.3.3 -setuptools==68.2.2 -simplejson==3.19.2 +setuptools==70.1.0 +simplejson==3.19.3 six==1.16.0 smmap==5.0.1 -sniffio==1.3.0 -SQLAlchemy==1.4.51 +sniffio==1.3.1 +SQLAlchemy==1.4.54 SQLAlchemy-JSONField==1.0.2 sqlalchemy-migrate==0.13.0 -SQLAlchemy-Utils==0.41.1 -sqlparse==0.4.4 +SQLAlchemy-Utils==0.41.2 +sqlparse==0.5.1 statsd==4.0.1 -stevedore==5.2.0 +stevedore==5.3.0 tabulate==0.9.0 Tempita==0.5.2 -tenacity==8.2.3 +tenacity==8.5.0 termcolor==2.4.0 testresources==2.0.1 testscenarios==0.5.0 -testtools==2.7.1 +testtools==2.7.2 text-unidecode==1.3 thrift==0.16.0 thrift-sasl==0.4.3 tiddlyweb==2.4.3 -time-machine==2.13.0 -tornado==6.4 -types-python-dateutil==2.8.19.20240106 -typing_extensions==4.9.0 +time-machine==2.15.0 +tornado==6.4.1 +types-python-dateutil==2.9.0.20240906 +typing_extensions==4.12.2 tzdata==2024.1 uc-micro-py==1.0.3 ulid==1.1 unicodecsv==0.14.1 -universal-pathlib==0.1.4 -urllib3==1.26.18 -uWSGI==2.0.24 +universal_pathlib==0.2.5 +urllib3==2.2.2 +uWSGI==2.0.27 vine==5.1.0 wcwidth==0.2.13 -WebOb==1.8.7 -websocket-client==1.7.0 +WebOb==1.8.8 +websocket-client==1.8.0 Werkzeug==2.2.3 -wheel==0.41.2 +wheel==0.43.0 +wirerope==0.4.7 wrapt==1.16.0 WTForms==3.1.2 xattr==0.10.1 yappi==1.6.0 -yarl==1.9.4 -zipp==3.17.0 +yarl==1.11.1 +zipp==3.20.2 diff --git a/src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py b/src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py index c40cb7a8..83838e65 100644 --- a/src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py +++ b/src/bin/shipyard_airflow/tests/unit/plugins/test_deployment_status_operator.py @@ -19,20 +19,24 @@ import yaml import kubernetes import airflow +import pendulum from shipyard_airflow.common.document_validators import \ document_validation_utils from shipyard_airflow.common.document_validators import errors from shipyard_airflow.plugins import deckhand_client_factory from shipyard_airflow.plugins import deployment_status_operator from shipyard_airflow.plugins import xcom_puller +from airflow import DAG +from airflow.models import TaskInstance, DagRun +from airflow.utils.dates import days_ago +from airflow.utils.state import State +from airflow.utils.types import DagRunType - -@mock.patch.object(airflow.models.BaseOperator, '__init__') +# @mock.patch.object(airflow.models.BaseOperator, '__init__') class TestDeploymentStatusOperator(TestCase): def __init__(self, *args, **kwargs): super(TestDeploymentStatusOperator, self).__init__(*args, **kwargs) - self.context = {'ti': None} self.status = {'status': "doc"} self.revision_id = "revision_id" self.version = {'version': "doc"} @@ -42,33 +46,51 @@ class TestDeploymentStatusOperator(TestCase): } self.config_map_data = {'release': yaml.safe_dump(full_data)} - @mock.patch.object(xcom_puller.XcomPuller, 'get_concurrency_status', - return_value=True) + def setUp(self): + # Create a DAG for the test using pendulum + self.dag = DAG(dag_id="example_dag", start_date=pendulum.now('UTC').add(days=-1)) + + # Create a DagRun to associate with the TaskInstance + self.dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=pendulum.now('UTC').add(days=-1), # Use pendulum for execution_date + run_id="manual__" + pendulum.now('UTC').to_iso8601_string(), + run_type=DagRunType.MANUAL, + state=State.RUNNING + ) + + # Create a TaskInstance with run_id + self.operator = deployment_status_operator.DeploymentStatusOperator( + shipyard_conf='conf', main_dag_name='name', dag=self.dag, task_id="task") + + self.task_instance = TaskInstance(task=self.operator, run_id=self.dag_run.run_id) + self.task_instance.dag_run = self.dag_run # Associate TaskInstance with DagRun + self.task_instance.state = State.RUNNING + + # Set the context to include TaskInstance + self.context = {'ti': self.task_instance} + + @mock.patch.object(deployment_status_operator.XcomPuller, 'get_concurrency_status', return_value=True) @mock.patch('shipyard_airflow.conf.config.parse_args') - @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, - '_get_version_doc') - @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, - '_get_status_and_revision') - @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, - '_store_as_config_map') + @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_get_version_doc') + @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_get_status_and_revision') + @mock.patch.object(deployment_status_operator.DeploymentStatusOperator, '_store_as_config_map') def test_execute(self, store_as_config_map, get_status_and_revision, get_version_doc, config_parse_args, - xcom_puller, - base_operator_init): + get_concurrency_status_mock): + # Mock return values for the methods get_status_and_revision.return_value = (self.status, self.revision_id) get_version_doc.return_value = self.version - operator = deployment_status_operator.DeploymentStatusOperator( - shipyard_conf='conf', main_dag_name='name') - - operator.execute(self.context) + # Execute the operator + self.operator.execute(self.context) assert config_parse_args.called - assert xcom_puller.called + assert get_concurrency_status_mock.called assert get_status_and_revision.called get_version_doc.assert_called_once_with(self.revision_id) store_as_config_map.assert_called_once_with(self.config_map_data) @@ -78,12 +100,10 @@ class TestDeploymentStatusOperator(TestCase): @mock.patch('shipyard_airflow.conf.config.parse_args') def test_execute_concurrency_fail(self, config_parse_args, - xcom_puller, - base_operator_init): - operator = deployment_status_operator.DeploymentStatusOperator( - shipyard_conf='conf', main_dag_name='name') + xcom_puller): + try: - operator.execute(self.context) + self.operator.execute(self.context) except airflow.AirflowException as err: assert str(err) == "Concurrency check did not pass, so the " \ "deployment status will not be updated" @@ -104,15 +124,13 @@ class TestDeploymentStatusOperator(TestCase): create_cfg_map_object, create_cfg_map, patch_cfg_map, - load_k8s_config, - base_operator_init): + load_k8s_config): data = 'data' patch_cfg_map.side_effect = kubernetes.client.rest.ApiException( status=404) - operator = deployment_status_operator.DeploymentStatusOperator( - shipyard_conf='conf', main_dag_name='name') - operator._store_as_config_map(data) + + self.operator._store_as_config_map(data) patch_cfg_map.assert_called_once_with( config_parser.return_value, config_parser.return_value, @@ -144,13 +162,11 @@ class TestDeploymentStatusOperator(TestCase): create_cfg_map, patch_cfg_map, load_k8s_config, - get_deployment_status, - base_operator_init): + get_deployment_status): data = 'data' - operator = deployment_status_operator.DeploymentStatusOperator( - shipyard_conf='conf', main_dag_name='name') - operator._store_as_config_map(data) + + self.operator._store_as_config_map(data) patch_cfg_map.assert_called_once_with( config_parser.return_value, config_parser.return_value, @@ -173,16 +189,14 @@ class TestDeploymentStatusOperator(TestCase): config_parser, create_cfg_map_object, patch_cfg_map, - load_k8s_config, - base_operator_init): + load_k8s_config): data = 'data' patch_cfg_map.side_effect = kubernetes.client.rest.ApiException( status=409) - operator = deployment_status_operator.DeploymentStatusOperator( - shipyard_conf='conf', main_dag_name='name') + try: - operator._store_as_config_map(data) + self.operator._store_as_config_map(data) except kubernetes.client.rest.ApiException as err: assert patch_cfg_map.side_effect == err @@ -206,11 +220,9 @@ class TestDeploymentStatusOperator(TestCase): def test__get_version_doc(self, config_parser, get_unique_doc, - get_client, - base_operator_init): - operator = deployment_status_operator.DeploymentStatusOperator( - shipyard_conf='conf', main_dag_name='name') - result = operator._get_version_doc(self.revision_id) + get_client): + + result = self.operator._get_version_doc(self.revision_id) assert result == get_unique_doc.return_value assert get_client.called @@ -229,13 +241,11 @@ class TestDeploymentStatusOperator(TestCase): def test__get_version_doc_does_not_exist(self, config_parser, get_unique_doc, - get_client, - base_operator_init): + get_client): get_unique_doc.side_effect = errors.DocumentNotFoundError() - operator = deployment_status_operator.DeploymentStatusOperator( - shipyard_conf='conf', main_dag_name='name') - result = operator._get_version_doc(self.revision_id) + + result = self.operator._get_version_doc(self.revision_id) assert result == {} assert get_client.called @@ -247,21 +257,18 @@ class TestDeploymentStatusOperator(TestCase): @mock.patch('shipyard_airflow.plugins.deployment_status_operator' '.get_deployment_status') def test__get_status_and_revision(self, - get_deployment_status, - base_operator_init): - operator = deployment_status_operator.DeploymentStatusOperator( - shipyard_conf='conf', main_dag_name='name') + get_deployment_status): action = {'committed_rev_id': self.revision_id} - operator.xcom_puller = mock.MagicMock() - operator.xcom_puller.get_action_info = mock.MagicMock() - operator.xcom_puller.get_action_info.return_value = action + self.operator.xcom_puller = mock.MagicMock() + self.operator.xcom_puller.get_action_info = mock.MagicMock() + self.operator.xcom_puller.get_action_info.return_value = action - status_and_revision = operator._get_status_and_revision() + status_and_revision = self.operator._get_status_and_revision() get_deployment_status.assert_called_once_with( action, - force_completed=operator.force_completed) + force_completed=self.operator.force_completed) assert status_and_revision == (get_deployment_status.return_value, self.revision_id) diff --git a/src/bin/shipyard_airflow/tox.ini b/src/bin/shipyard_airflow/tox.ini index 27819e07..dc81b771 100644 --- a/src/bin/shipyard_airflow/tox.ini +++ b/src/bin/shipyard_airflow/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist=True -envlist = py38, pep8, bandit +envlist = py310, pep8, bandit [testenv] setenv= @@ -17,20 +17,30 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt - -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt commands= rm -f {toxinidir}/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" -[testenv:py38] +[testenv:safety] +basepython = python3 +deps = + safety +allowlist_externals= + safety +commands = + safety check -r {toxinidir}/requirements-frozen.txt --full-report + +[testenv:py310] skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning -basepython=python3.8 +basepython=python3.10 allowlist_externals= bash airflow + ../../../tools/wait_for_dag.sh commands = bash -c "rm -rf $HOME/airflow" airflow version @@ -39,9 +49,14 @@ commands = airflow info airflow dags list airflow dags list-import-errors + bash -c "nohup airflow scheduler >/dev/null 2>&1 &" + ../../../tools/wait_for_dag.sh + airflow dags list + airflow dags unpause example_bash_operator airflow tasks test example_bash_operator runme_0 airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 airflow tasks run example_bash_operator runme_0 2018-01-01 + airflow tasks states-for-dag-run example_bash_operator backfill__2018-01-01T00:00:00+00:00 airflow dags state example_bash_operator 2018-01-01 pytest {posargs} -vv @@ -53,6 +68,7 @@ setenv= allowlist_externals= bash airflow + ../../../tools/wait_for_dag.sh commands = bash -c "rm -rf $HOME/airflow" airflow version @@ -61,9 +77,15 @@ commands = airflow info airflow dags list airflow dags list-import-errors + bash -c "nohup airflow scheduler >/dev/null 2>&1 &" + ../../../tools/wait_for_dag.sh + airflow dags list + airflow dags unpause example_bash_operator airflow tasks test example_bash_operator runme_0 airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 airflow tasks run example_bash_operator runme_0 2018-01-01 + airflow tasks states-for-dag-run example_bash_operator backfill__2018-01-02T00:00:00+00:00 + airflow dags state example_bash_operator 2018-01-01 pytest \ {posargs} \ --cov-branch \ diff --git a/src/bin/shipyard_client/requirements-frozen.txt b/src/bin/shipyard_client/requirements-frozen.txt index 8ad50451..c524b9c0 100644 --- a/src/bin/shipyard_client/requirements-frozen.txt +++ b/src/bin/shipyard_client/requirements-frozen.txt @@ -1,43 +1,41 @@ arrow==1.3.0 -certifi==2024.2.2 +certifi==2024.8.30 charset-normalizer==3.3.2 click==8.1.7 click-default-group==1.2.4 debtcollector==3.0.0 Flask==2.2.5 -idna==3.6 -importlib-metadata==6.11.0 +idna==3.10 iso8601==2.1.0 -itsdangerous==2.1.2 -Jinja2==3.1.3 +itsdangerous==2.2.0 +Jinja2==3.1.4 keystoneauth1==5.1.2 MarkupSafe==2.1.5 -msgpack==1.0.7 -netaddr==1.2.1 +msgpack==1.1.0 +netaddr==1.3.0 netifaces==0.11.0 os-service-types==1.7.0 oslo.config==9.1.1 oslo.i18n==6.0.0 oslo.serialization==5.1.1 oslo.utils==6.1.0 -packaging==23.2 -pbr==6.0.0 -pip==23.2.1 +packaging==24.1 +pbr==6.1.0 +pip==24.1 PTable==0.9.2 pylibyaml==0.1.0 -pyparsing==3.1.1 -python-dateutil==2.8.2 +pyparsing==3.1.4 +python-dateutil==2.9.0.post0 python-keystoneclient==5.1.0 -pytz==2024.1 -PyYAML==6.0.1 -requests==2.31.0 -rfc3986==1.5.0 -setuptools==68.2.2 +pytz==2024.2 +PyYAML==6.0.2 +requests==2.32.3 +rfc3986==2.0.0 +setuptools==70.1.0 six==1.16.0 -stevedore==5.2.0 -types-python-dateutil==2.8.19.20240106 -urllib3==1.26.18 +stevedore==5.3.0 +types-python-dateutil==2.9.0.20240906 +urllib3==2.2.2 Werkzeug==2.2.3 -wheel==0.41.2 +wheel==0.43.0 wrapt==1.16.0 -zipp==3.17.0 diff --git a/src/bin/shipyard_client/tox.ini b/src/bin/shipyard_client/tox.ini index c8e8d02f..2d08d4cf 100644 --- a/src/bin/shipyard_client/tox.ini +++ b/src/bin/shipyard_client/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist=True -envlist = py38, pep8, bandit +envlist = py310, pep8, bandit [testenv] setenv= @@ -18,12 +18,22 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt - -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt commands= rm -f {toxinidir}/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" -[testenv:py38] +[testenv:safety] +basepython = python3 +deps = + safety +allowlist_externals= + safety +commands = + safety check -r {toxinidir}/requirements-frozen.txt --full-report + + +[testenv:py310] skipsdist=True commands = pytest \ diff --git a/tools/airflow_image_run.sh b/tools/airflow_image_run.sh index 6b19b284..9169bef4 100755 --- a/tools/airflow_image_run.sh +++ b/tools/airflow_image_run.sh @@ -32,6 +32,7 @@ fi if [ "${USE_PROXY}" == "true" ]; then TEST_RESP="$(docker run \ -p 8080:8080 \ + --rm \ --env HTTP_PROXY="${PROXY}" \ --env HTTPS_PROXY="${PROXY}" \ --env NO_PROXY="${NO_PROXY}" \ @@ -40,12 +41,11 @@ if [ "${USE_PROXY}" == "true" ]; then else TEST_RESP="$(docker run \ -p 8080:8080 \ + --rm \ --name airflow_test ${IMAGE} \ quicktest)" fi -docker stop airflow_test -docker rm airflow_test if [ ${TEST_RESP:(-7)} == "success" ]; then diff --git a/tools/gate/playbooks/checkout-treasuremap-ref.yaml b/tools/gate/playbooks/checkout-treasuremap-ref.yaml new file mode 100644 index 00000000..46a24051 --- /dev/null +++ b/tools/gate/playbooks/checkout-treasuremap-ref.yaml @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + + tasks: + - name: Checkout treasuremap ref + shell: | + set -xe; + + : "${TREASUREMAP_REF:=v1.9}" + + cd ../treasuremap + git fetch https://review.opendev.org/airship/treasuremap ${TREASUREMAP_REF} && git checkout FETCH_HEAD + + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + TREASUREMAP_REF: "{{ treasuremap_ref }}" + +... \ No newline at end of file diff --git a/tools/gate/playbooks/prepare-hosts.yaml b/tools/gate/playbooks/prepare-hosts.yaml index e7e3d9c9..76087fde 100644 --- a/tools/gate/playbooks/prepare-hosts.yaml +++ b/tools/gate/playbooks/prepare-hosts.yaml @@ -15,4 +15,7 @@ roles: - bindep - start-zuul-console + - role: add-authorized-keys + public_keys: + - public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com ... diff --git a/tools/gate/roles/airship-run-script-set/defaults/main.yaml b/tools/gate/roles/airship-run-script-set/defaults/main.yaml index 725f4b23..7dc55afd 100644 --- a/tools/gate/roles/airship-run-script-set/defaults/main.yaml +++ b/tools/gate/roles/airship-run-script-set/defaults/main.yaml @@ -19,7 +19,7 @@ osh_params: container_distro_version: focal # feature_gates: site: airskiff -HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz +HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7 diff --git a/tools/gate/roles/airship-run-script-set/tasks/main.yaml b/tools/gate/roles/airship-run-script-set/tasks/main.yaml index 4129a2c1..0fe778cd 100644 --- a/tools/gate/roles/airship-run-script-set/tasks/main.yaml +++ b/tools/gate/roles/airship-run-script-set/tasks/main.yaml @@ -35,7 +35,7 @@ FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}" RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}" PL_SITE: "{{ site | default('airskiff') }}" - HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}" + HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz') }}" HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}" OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}" OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}" diff --git a/tools/gate/roles/airship-run-script/defaults/main.yaml b/tools/gate/roles/airship-run-script/defaults/main.yaml index 725f4b23..7dc55afd 100644 --- a/tools/gate/roles/airship-run-script/defaults/main.yaml +++ b/tools/gate/roles/airship-run-script/defaults/main.yaml @@ -19,7 +19,7 @@ osh_params: container_distro_version: focal # feature_gates: site: airskiff -HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz +HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7 diff --git a/tools/gate/roles/airship-run-script/tasks/main.yaml b/tools/gate/roles/airship-run-script/tasks/main.yaml index 006106b3..0220ef49 100644 --- a/tools/gate/roles/airship-run-script/tasks/main.yaml +++ b/tools/gate/roles/airship-run-script/tasks/main.yaml @@ -32,7 +32,7 @@ FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}" RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}" PL_SITE: "{{ site | default('airskiff') }}" - HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}" + HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz') }}" HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}" OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}" OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}" diff --git a/tools/helm_install.sh b/tools/helm_install.sh index 9c0dae59..77171564 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,7 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"} +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz"} function install_helm_binary { diff --git a/tools/wait_for_dag.sh b/tools/wait_for_dag.sh new file mode 100755 index 00000000..81f11c2e --- /dev/null +++ b/tools/wait_for_dag.sh @@ -0,0 +1,10 @@ +#!/bin/bash +while true; do + is_active=$(airflow dags details example_bash_operator -o plain | grep is_active | awk '{print $2}') + if [ "$is_active" == "True" ]; then + echo "DAG example_bash_operator is active." + break + fi + echo "Waiting for DAG example_bash_operator to become active..." + sleep 10 +done \ No newline at end of file diff --git a/tox.ini b/tox.ini index 7c0bfb50..40e1aaea 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,27 @@ passenv= [testenv:venv] commands = {posargs} +[testenv:airflow_safety] +basepython = python3 +deps = + safety +allowlist_externals= + safety + tox +commands = + tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e safety + +[testenv:shipyard_safety] +basepython = python3 +deps = + safety +allowlist_externals= + safety + tox +commands = + tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e safety + + # Please use `make docs` instead [testenv:docs] allowlist_externals= @@ -36,22 +57,22 @@ commands= tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -[testenv:py38] -deps = - tox<=4.0.0 +[testenv:py310] +; deps = +; tox allowlist_externals= tox bash commands= - tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38 - tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38 + tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py310 + tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py310 tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample" bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample" [testenv:cover] -deps = - tox<=4.0.0 +; deps = +; tox allowlist_externals= tox sh