From 146f1debbaf85f66e22cf8c546c3f343f0180ec9 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Thu, 29 Aug 2024 00:37:52 +0000 Subject: [PATCH] Airflow 2.10.2 + ubuntu_jammy Change-Id: Ie498540c506060224d13d2af6689bb04770b32e9 --- .zuul.yaml | 55 ++++++------ Makefile | 12 ++- doc/source/_static/drydock.conf.sample | 2 +- etc/drydock/drydock.conf.sample | 2 +- images/drydock/Dockerfile.ubuntu_jammy | 116 +++++++++++++++++++++++++ python/requirements-frozen.txt | 102 ++++++++++------------ tools/helm_install.sh | 2 +- tox.ini | 6 +- 8 files changed, 209 insertions(+), 88 deletions(-) create mode 100644 images/drydock/Dockerfile.ubuntu_jammy diff --git a/.zuul.yaml b/.zuul.yaml index 50e74d5b..c9d053fc 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,30 +13,31 @@ - project: templates: - docs-on-readthedocs - - openstack-python38-jobs vars: rtd_webhook_id: '38809' rtd_project_name: 'airship-drydock' check: jobs: - - airship-drydock-openstack-tox-cover-focal - - airship-drydock-openstack-tox-pep8-focal + - openstack-tox-py310 + - airship-drydock-openstack-tox-cover-jammy + - airship-drydock-openstack-tox-pep8-jammy - airship-drydock-baclient-test - airship-drydock-chart-build-gate - airship-drydock-chart-build-latest-htk - - airship-drydock-docker-build-gate-ubuntu_focal + - airship-drydock-docker-build-gate-ubuntu_jammy gate: jobs: - - airship-drydock-openstack-tox-cover-focal - - airship-drydock-openstack-tox-pep8-focal + - openstack-tox-py310 + - airship-drydock-openstack-tox-cover-jammy + - airship-drydock-openstack-tox-pep8-jammy - airship-drydock-baclient-test - airship-drydock-chart-build-gate - - airship-drydock-docker-build-gate-ubuntu_focal + - airship-drydock-docker-build-gate-ubuntu_jammy post: jobs: - - airship-drydock-docker-publish-ubuntu_focal + - airship-drydock-docker-publish-ubuntu_jammy - drydock-upload-git-mirror - nodeset: @@ -46,24 +47,24 @@ label: ubuntu-bionic - nodeset: - name: airship-drydock-single-node-focal + name: airship-drydock-single-node-jammy nodes: - name: primary - label: ubuntu-focal + label: ubuntu-jammy - job: - name: airship-drydock-openstack-tox-cover-focal + name: airship-drydock-openstack-tox-cover-jammy parent: openstack-tox-cover - description: Runs cover job on focal - nodeset: airship-drydock-single-node-focal + description: Runs cover job on jammy + nodeset: airship-drydock-single-node-jammy pre-run: tools/gate/playbooks/install-docker.yaml - job: - name: airship-drydock-openstack-tox-pep8-focal + name: airship-drydock-openstack-tox-pep8-jammy parent: openstack-tox-pep8 - description: Runs pep8 job on focal - nodeset: airship-drydock-single-node-focal + description: Runs pep8 job on jammy + nodeset: airship-drydock-single-node-jammy - job: name: airship-drydock-baclient-test @@ -71,7 +72,7 @@ Run a bootaction drydock client test run: tools/gate/playbooks/baclient_test.yaml timeout: 3600 - nodeset: airship-drydock-single-node-focal + nodeset: airship-drydock-single-node-jammy - job: name: airship-drydock-chart-build-gate @@ -79,9 +80,9 @@ Builds charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: airship-drydock-single-node-focal + nodeset: airship-drydock-single-node-jammy vars: - HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f + HTK_COMMIT: 43fd7143481b6ddda0dbd2f26bf6ec39a417b15b - job: name: airship-drydock-chart-build-latest-htk @@ -90,21 +91,21 @@ timeout: 900 voting: false run: tools/gate/playbooks/build-charts.yaml - nodeset: airship-drydock-single-node-focal + nodeset: airship-drydock-single-node-jammy vars: HTK_COMMIT: master - job: - name: airship-drydock-docker-build-gate-ubuntu_focal + name: airship-drydock-docker-build-gate-ubuntu_jammy timeout: 3600 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: airship-drydock-single-node-focal + nodeset: airship-drydock-single-node-jammy irrelevant-files: - '^doc/.*' - '^charts/.*' vars: publish: false - distro: ubuntu_focal + distro: ubuntu_jammy tags: dynamic: patch_set: true @@ -112,10 +113,10 @@ - job: - name: airship-drydock-docker-publish-ubuntu_focal + name: airship-drydock-docker-publish-ubuntu_jammy timeout: 3600 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: airship-drydock-single-node-focal + nodeset: airship-drydock-single-node-jammy secrets: - airship_drydock_quay_creds irrelevant-files: @@ -123,14 +124,14 @@ - '^charts/.*' vars: publish: true - distro: ubuntu_focal + distro: ubuntu_jammy tags: dynamic: branch: true commit: true static: - latest - - airflow_2.8.2 + - airflow_2.10.0 - secret: diff --git a/Makefile b/Makefile index 82d3d80c..03b60869 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,8 @@ IMAGE_PREFIX ?= airshipit IMAGE_TAG ?= latest HELM := $(shell realpath $(BUILD_DIR))/helm UBUNTU_BASE_IMAGE ?= -DISTRO ?= ubuntu_focal +DISTRO ?= ubuntu_jammy +DISTRO_ALIAS ?= ubuntu_focal PROXY ?= http://proxy.foo.com:8000 NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local USE_PROXY ?= false @@ -28,6 +29,7 @@ PUSH_IMAGE ?= false LABEL ?= org.airshipit.build=community COMMIT ?= $(shell git rev-parse HEAD) IMAGE ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} +IMAGE_ALIAS := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO_ALIAS} export @@ -88,6 +90,14 @@ helm-install: build_drydock: export; tools/drydock_image_build.sh + +ifneq ($(DISTRO), $(DISTRO_ALIAS)) + docker tag $(IMAGE) $(IMAGE_ALIAS) +endif +ifeq ($(DOCKER_REGISTRY), localhost:5000) + docker push $(IMAGE) + docker push $(IMAGE_ALIAS) +endif ifeq ($(PUSH_IMAGE), true) docker push $(IMAGE) endif diff --git a/doc/source/_static/drydock.conf.sample b/doc/source/_static/drydock.conf.sample index b04f5056..323d4df9 100644 --- a/doc/source/_static/drydock.conf.sample +++ b/doc/source/_static/drydock.conf.sample @@ -78,7 +78,7 @@ # Domain name containing project (string value) #project_domain_name = -# Trust ID (string value) +# ID of the trust to use as a trustee use (string value) #trust_id = # Optional domain ID to use with v3 and v2 parameters. It will be used for both diff --git a/etc/drydock/drydock.conf.sample b/etc/drydock/drydock.conf.sample index b04f5056..323d4df9 100644 --- a/etc/drydock/drydock.conf.sample +++ b/etc/drydock/drydock.conf.sample @@ -78,7 +78,7 @@ # Domain name containing project (string value) #project_domain_name = -# Trust ID (string value) +# ID of the trust to use as a trustee use (string value) #trust_id = # Optional domain ID to use with v3 and v2 parameters. It will be used for both diff --git a/images/drydock/Dockerfile.ubuntu_jammy b/images/drydock/Dockerfile.ubuntu_jammy new file mode 100644 index 00000000..827b6dbd --- /dev/null +++ b/images/drydock/Dockerfile.ubuntu_jammy @@ -0,0 +1,116 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG FROM=ubuntu:jammy + +FROM golang:1.22.7 as baclient_builder + +COPY ./tools/baclient_build.sh /tmp/drydock/ +COPY ./go /tmp/drydock/go + +WORKDIR /tmp/drydock +RUN PATH=$PATH:/usr/local/go/bin ./baclient_build.sh /tmp/drydock/go /tmp/drydock/baclient + +FROM ${FROM} AS libyaml_builder + +# Build LibYAML +ARG LIBYAML_VERSION=0.2.5 +RUN set -ex \ + && apt-get update && apt-get upgrade -y \ + && apt install -y git automake make libtool \ + && git clone https://github.com/yaml/libyaml.git \ + && cd libyaml \ + && git checkout $LIBYAML_VERSION \ + && ./bootstrap \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -fr libyaml + +RUN rm -r /var/lib/apt/lists/* + + +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://airship-drydock.readthedocs.org' \ + org.opencontainers.image.source='https://git.openstack.org/openstack/airship-drydock' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +ARG UBUNTU_REPO=http://archive.ubuntu.com/ubuntu +ARG TRUSTED_UBUNTU_REPO=no +ARG ALLOW_UNAUTHENTICATED=false +ARG PIP_TRUSTED_HOST=foo.com +ARG PIP_INDEX_URL=https://pypi.org/simple +ARG BUILD_DIR +ENV container docker +ENV PORT 9000 +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + + +# COPY ./bindep-python.txt /tmp/drydock/ +WORKDIR /tmp/drydock +RUN DEBIAN_FRONTEND=noninteractive \ + apt update \ + && DEBIAN_FRONTEND=noninteractive \ + apt install -y \ + --allow-downgrades \ + --no-install-recommends \ + python3-dev \ + python3-pip \ + python3-setuptools \ + pkg-config \ + libvirt-dev \ + libssl-dev \ + libpq-dev \ + gcc \ + openssh-client\ + curl \ + netbase \ + && rm -r /var/lib/apt/lists/* + +# Install LibYAML +ENV LD_LIBRARY_PATH=/usr/local/lib +COPY --from=libyaml_builder /usr/local/lib /usr/local/lib +COPY --from=libyaml_builder /usr/local/include/yaml.h /usr/local/include/yaml.h + +COPY ./python/requirements-frozen.txt /tmp/drydock/ +RUN \ + pip3 install --upgrade pip \ + && pip3 install \ + --no-cache-dir \ + -r /tmp/drydock/requirements-frozen.txt + +COPY ./python /tmp/drydock/python +WORKDIR /tmp/drydock/python +RUN cd /tmp/drydock/python \ + && pip3 install $(pwd) + +COPY ./alembic /tmp/drydock/alembic +COPY ./alembic.ini /tmp/drydock/alembic.ini +COPY ./entrypoint.sh /tmp/drydock/entrypoint.sh + +COPY --from=baclient_builder /tmp/drydock/baclient /tmp/drydock/python/drydock_provisioner/assets/baclient + + +EXPOSE $PORT + +WORKDIR /tmp/drydock + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["server"] diff --git a/python/requirements-frozen.txt b/python/requirements-frozen.txt index 3fa316eb..ea0c871d 100644 --- a/python/requirements-frozen.txt +++ b/python/requirements-frozen.txt @@ -1,45 +1,42 @@ -alembic==1.13.1 +alembic==1.13.2 amqp==5.2.0 -attrs==23.2.0 -backports.zoneinfo==0.2.1 -bcrypt==4.1.2 -Beaker==1.12.1 -cachetools==5.3.2 -certifi==2024.2.2 -cffi==1.16.0 +attrs==24.2.0 +bcrypt==4.2.0 +Beaker==1.13.0 +cachetools==5.5.0 +certifi==2024.8.30 +cffi==1.17.1 charset-normalizer==3.3.2 click==8.1.7 -cryptography==41.0.7 +cryptography==42.0.8 debtcollector==3.0.0 decorator==5.1.1 defusedxml==0.7.1 dnspython==2.6.1 -dogpile.cache==1.3.2 -eventlet==0.35.2 +dogpile.cache==1.3.3 +eventlet==0.37.0 falcon==3.1.3 fasteners==0.19 fixtures==4.1.0 futurist==3.0.0 -greenlet==3.0.3 -idna==3.6 -importlib-metadata==6.11.0 -importlib-resources==5.13.0 +greenlet==3.1.0 +idna==3.10 iso8601==2.1.0 -Jinja2==3.1.3 +Jinja2==3.1.4 jsonpatch==1.33 jsonpath-rw==1.4.0 -jsonpointer==2.4 -jsonschema==4.21.1 +jsonpointer==3.0.0 +jsonschema==4.23.0 jsonschema-specifications==2023.12.1 keystoneauth1==5.1.2 keystonemiddleware==10.2.0 -kombu==5.3.5 -libvirt-python==10.0.0 -Mako==1.3.2 +kombu==5.4.1 +libvirt-python==10.7.0 +Mako==1.3.5 MarkupSafe==2.1.5 mock==5.1.0 -msgpack==1.0.7 -netaddr==1.2.1 +msgpack==1.1.0 +netaddr==1.3.0 netifaces==0.11.0 oauthlib==3.2.2 os-service-types==1.7.0 @@ -59,58 +56,55 @@ oslo.service==3.1.1 oslo.upgradecheck==2.1.1 oslo.utils==6.1.0 oslo.versionedobjects==3.1.0 -packaging==23.2 -Paste==3.7.1 +packaging==24.1 +Paste==3.10.1 PasteDeploy==3.1.0 -pbr==6.0.0 -pip==23.2.1 -pkgutil_resolve_name==1.3.10 +pbr==6.1.0 +pip==24.1 ply==3.11 -prettytable==3.10.0 +prettytable==3.11.0 prometheus_client==0.20.0 psycopg2-binary==2.9.9 PTable==0.9.2 pycadf==3.1.1 -pycparser==2.21 -pyghmi==1.5.67 +pycparser==2.22 +pyghmi==1.5.72 pylibyaml==0.1.0 -pymongo==4.6.2 -pyparsing==3.1.1 -python-dateutil==2.8.2 +pymongo==4.8.0 +pyparsing==3.1.4 +python-dateutil==2.9.0.post0 python-keystoneclient==5.1.0 python-memcached==1.62 -pytz==2024.1 -PyYAML==6.0.1 -redfish==3.2.2 -referencing==0.33.0 +pytz==2024.2 +PyYAML==6.0.2 +redfish==3.2.5 +referencing==0.35.1 repoze.lru==0.7 -requests==2.31.0 +requests==2.32.3 requests-toolbelt==1.0.0 requests-unixsocket==0.3.0 -responses==0.25.0 -rfc3986==1.5.0 +responses==0.25.3 +rfc3986==2.0.0 Routes==2.5.1 -rpds-py==0.18.0 -setuptools==68.2.2 +rpds-py==0.20.0 +setuptools==70.1.0 six==1.16.0 -SQLAlchemy==1.4.51 +SQLAlchemy==1.4.54 sqlalchemy-migrate==0.13.0 -sqlparse==0.4.4 +sqlparse==0.5.1 statsd==4.0.1 -stevedore==5.2.0 +stevedore==5.3.0 Tempita==0.5.2 testresources==2.0.1 testscenarios==0.5.0 -testtools==2.7.1 -typing_extensions==4.9.0 -tzdata==2024.1 +testtools==2.7.2 +typing_extensions==4.12.2 ulid2==0.3.0 -urllib3==1.26.18 -uWSGI==2.0.24 +urllib3==2.2.2 +uWSGI==2.0.27 vine==5.1.0 wcwidth==0.2.13 -WebOb==1.8.7 -wheel==0.41.2 +WebOb==1.8.8 +wheel==0.43.0 wrapt==1.16.0 yappi==1.6.0 -zipp==3.17.0 diff --git a/tools/helm_install.sh b/tools/helm_install.sh index 81eddd2b..1cebfa21 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,7 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"} +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz"} function install_helm_binary { diff --git a/tox.ini b/tox.ini index b2d4c1b5..63e976d2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38,pep8,bandit +envlist = py310,pep8,bandit setupdir=python/ minversion = 3.28.0 @@ -20,7 +20,7 @@ allowlist_externals= sh deps= -r{toxinidir}/python/requirements-direct.txt - -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.8.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt commands= rm -f python/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen.txt" @@ -34,7 +34,7 @@ commands= yapf -i -r --style=pep8 {toxinidir}/python/tests find {toxinidir}/python/drydock_provisioner -name '__init__.py' -exec yapf -i --style=pep8 \{\} ; -[testenv:py38] +[testenv:py310] usedevelop=True allowlist_externals= py.test