diff --git a/.zuul.yaml b/.zuul.yaml index aea9a16a..4e11edcb 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -14,6 +14,7 @@ templates: - docs-on-readthedocs - openstack-python38-jobs + - openstack-python36-jobs vars: rtd_webhook_id: '38576' rtd_project_name: 'airship-shipyard' @@ -21,20 +22,26 @@ check: jobs: - openstack-tox-pep8 + - airship-shipyard-openstack-tox-cover-focal - airship-shipyard-chart-build-gate - airship-shipyard-chart-build-latest-htk - airship-shipyard-whitespace-lint-gate - airship-shipyard-airskiff-deployment - airship-shipyard-image-gate-ubuntu_focal + - airship-shipyard-image-gate-ubuntu_bionic gate: jobs: - openstack-tox-pep8 + - airship-shipyard-openstack-tox-cover-focal - airship-shipyard-chart-build-gate - airship-shipyard-whitespace-lint-gate + - airship-shipyard-airskiff-deployment - airship-shipyard-image-gate-ubuntu_focal + - airship-shipyard-image-gate-ubuntu_bionic post: jobs: - - airship-shipyard-docker-build-post-ubuntu_focal + - airship-shipyard-docker-build-publish-ubuntu_focal + - airship-shipyard-docker-build-publish-ubuntu_bionic - shipyard-upload-git-mirror - nodeset: @@ -49,12 +56,21 @@ - name: primary label: ubuntu-focal + +- job: + name: airship-shipyard-openstack-tox-cover-focal + parent: openstack-tox-cover + description: Runs cover job on focal + nodeset: airship-shipyard-single-node-focal + - job: name: airship-shipyard-chart-build-gate description: Build charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml nodeset: airship-shipyard-single-node-focal + vars: + HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b - job: name: airship-shipyard-chart-build-latest-htk @@ -78,13 +94,14 @@ description: | Base job for running airship-shipyard image related jobs. timeout: 3600 - run: tools/gate/playbooks/run-image.yaml + run: tools/gate/playbooks/docker-image-build.yaml nodeset: airship-shipyard-single-node-focal - irrelevant-files: + irrelevant-files: &irrelevant-files - ^.*\.rst$ - ^doc/.*$ - ^releasenotes/.*$ + - job: name: airship-shipyard-image-gate-ubuntu_focal description: | @@ -97,6 +114,19 @@ dynamic: patch_set: true +- job: + name: airship-shipyard-image-gate-ubuntu_bionic + description: | + Run shipyard-image build for ubuntu_bionic + parent: airship-shipyard-image-base + irrelevant-files: *irrelevant-files + vars: + publish: false + distro: ubuntu_bionic + tags: + dynamic: + patch_set: true + - job: name: airship-shipyard-airskiff-deployment nodeset: airship-shipyard-single-node-focal @@ -114,29 +144,14 @@ vars: CLONE_SHIPYARD: false HTK_COMMIT: master - irrelevant-files: &irrelevant-files - - ^.*\.rst$ - - ^doc/.*$ - - ^releasenotes/.*$ - -- job: - name: airship-shipyard-image-gate-opensuse - description: | - Run shipyard-image build for opensuse. - parent: airship-shipyard-image-base - vars: - publish: false - distro: opensuse_15 - tags: - dynamic: - patch_set: true + irrelevant-files: *irrelevant-files - job: - name: airship-shipyard-docker-build-post-ubuntu_focal + name: airship-shipyard-docker-build-publish-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/run-image.yaml - nodeset: airship-shipyard-single-node + nodeset: airship-shipyard-single-node-focal irrelevant-files: *irrelevant-files secrets: - airship_shipyard_quay_creds @@ -151,19 +166,16 @@ - latest - job: - name: airship-shipyard-docker-build-post-opensuse + name: airship-shipyard-docker-build-publish-ubuntu_bionic timeout: 1800 run: tools/gate/playbooks/run-image.yaml - nodeset: airship-shipyard-single-node - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^releasenotes/.*$ + nodeset: airship-shipyard-single-node-focal + irrelevant-files: *irrelevant-files secrets: - airship_shipyard_quay_creds vars: publish: true - distro: opensuse_15 + distro: ubuntu_bionic tags: dynamic: branch: true @@ -171,6 +183,7 @@ static: - latest + - secret: name: airship_shipyard_quay_creds data: @@ -205,15 +218,15 @@ git_mirror_repository: airshipit/shipyard secrets: - name: git_mirror_credentials - secret: shipyard-airshipit-github-secret + secret: shipyard-airshipit-github-secret-2023-03-23 pass-to-parent: true - secret: - name: shipyard-airshipit-github-secret + name: shipyard-airshipit-github-secret-2023-03-23 data: user: git host: github.com - host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== + host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= ssh_key: !encrypted/pkcs1-oaep - hUsE3gOWqb6aoNVSpqRHcUbqi0wOjRJ47viZWr7JopICoyhIerZ6DN4dGRaFjJ+8UtQvD uXKYHZ35+YbsJDQttIadtVvj5Va8hIToI/rARSktyDp4zb+fK829INKuOjSszowhGoDuv diff --git a/Makefile b/Makefile index 9dd33cf0..375a3fc7 100644 --- a/Makefile +++ b/Makefile @@ -40,18 +40,36 @@ IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO} IMAGE_DIR:=images/$(IMAGE_NAME) .PHONY: images -#Build all images in the list -images: $(IMAGE_NAME) -#Build and run all images in list -#sudo make images IMAGE_NAME=airflow will Build and Run airflow -#sudo make images will build and run airflow and shipyard -$(IMAGE_NAME): - @echo - @echo "===== Processing [$@] image =====" - @make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/$@ IMAGE_NAME=$@ - @make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/$@_image_run.sh # Build all docker images for this project +images: build_images + +build_images: build_airflow build_shipyard + +run_images: build_airflow run_airflow build_shipyard run_shipyard + +#Build all images in list +build_airflow: + @echo + @echo "===== Processing [airflow] image =====" + @make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/airflow:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/airflow IMAGE_NAME=airflow + +build_shipyard: + @echo + @echo "===== Processing [shipyard] image =====" + @make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/shipyard:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/shipyard IMAGE_NAME=shipyard + + +#Run all images in list + +run_airflow: + @echo + @echo "===== Processing [airflow] image =====" + @make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/airflow:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/airflow_image_run.sh +run_shipyard: + @echo + @echo "===== Processing [shipyard] image =====" + @make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/shipyard:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/shipyard_image_run.sh # Create tgz of the chart .PHONY: charts diff --git a/images/airflow/Dockerfile.ubuntu_bionic b/images/airflow/Dockerfile.ubuntu_bionic index 640430fd..b4134a21 100644 --- a/images/airflow/Dockerfile.ubuntu_bionic +++ b/images/airflow/Dockerfile.ubuntu_bionic @@ -64,10 +64,13 @@ RUN set -ex && \ libffi-dev \ libssl-dev \ libpq-dev \ + libpq5 \ libtool \ locales \ netcat \ netbase \ + postgresql-client \ + postgresql-common \ python3 \ python3-setuptools \ python3-pip \ @@ -110,7 +113,7 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \ # Note - removing snakebite (python 2 vs. 3). See: # https://github.com/puckel/docker-airflow/issues/77 # Install Airflow directly to allow overriding source -COPY images/airflow/requirements.txt /tmp/ +COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /tmp/requirements.txt RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \ && pip3 install $AIRFLOW_SRC --no-cache-dir \ && (pip3 uninstall -y snakebite || true) \ @@ -135,8 +138,8 @@ ENV PBR_VERSION 0.1a1 # Since Shipyard and Airflow are built together as images, this should prevent # stale or out-of-date code between these parts. # Shipyard requirements, source and installation -COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt -RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir +# COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /tmp/api_requirements.txt +# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/ RUN cd /tmp/shipyard \ diff --git a/images/airflow/Dockerfile.ubuntu_focal b/images/airflow/Dockerfile.ubuntu_focal index 5a6bcf5f..42415b9f 100644 --- a/images/airflow/Dockerfile.ubuntu_focal +++ b/images/airflow/Dockerfile.ubuntu_focal @@ -148,8 +148,8 @@ ENV PBR_VERSION 0.1a1 # Since Shipyard and Airflow are built together as images, this should prevent # stale or out-of-date code between these parts. # Shipyard requirements, source and installation -COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt -RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir +# COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt +# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/ RUN cd /tmp/shipyard \ diff --git a/images/shipyard/Dockerfile.ubuntu_bionic b/images/shipyard/Dockerfile.ubuntu_bionic index e6dfd600..77443cd3 100644 --- a/images/shipyard/Dockerfile.ubuntu_bionic +++ b/images/shipyard/Dockerfile.ubuntu_bionic @@ -85,8 +85,8 @@ RUN chown -R shipyard: /home/shipyard \ && chmod +x /home/shipyard/entrypoint.sh # Requirements and Shipyard source -COPY ${ctx_base}/shipyard_airflow/requirements.txt /home/shipyard/api_requirements.txt -COPY ${ctx_base}/shipyard_client/requirements.txt /home/shipyard/client_requirements.txt +COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /home/shipyard/api_requirements.txt +COPY ${ctx_base}/shipyard_client/requirements-frozen-bionic.txt /home/shipyard/client_requirements.txt COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/ diff --git a/src/bin/shipyard_airflow/requirements-direct.txt b/src/bin/shipyard_airflow/requirements-direct.txt index 4a2c0b7e..f6c053dd 100644 --- a/src/bin/shipyard_airflow/requirements-direct.txt +++ b/src/bin/shipyard_airflow/requirements-direct.txt @@ -34,7 +34,7 @@ WTForms<=2.3.3 itsdangerous==2.0.1 -git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e#egg=deckhand -git+https://opendev.org/airship/drydock.git@375abedb8aa3a413ca70a47aef467f39d65e1aee#egg=drydock_provisioner&subdirectory=python -git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97#egg=armada -git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c#egg=promenade +git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand +git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#egg=drydock_provisioner&subdirectory=python +git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f#egg=armada +git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb#egg=promenade diff --git a/src/bin/shipyard_airflow/requirements-frozen-bionic.txt b/src/bin/shipyard_airflow/requirements-frozen-bionic.txt new file mode 100644 index 00000000..ab99742d --- /dev/null +++ b/src/bin/shipyard_airflow/requirements-frozen-bionic.txt @@ -0,0 +1,241 @@ +alabaster==0.7.13 +alembic==1.0.1 +amqp==2.6.1 +apache-airflow==1.10.5 +apispec==1.3.3 +argcomplete==3.0.8 +Armada @ git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f +arrow==0.17.0 +attrs==22.2.0 +autopage==0.5.1 +Babel==2.11.0 +bcrypt==4.0.1 +Beaker==1.12.0 +billiard==3.6.4.0 +cached-property==1.5.2 +cachetools==4.2.4 +celery==4.3.0 +certifi==2023.5.7 +cffi==1.15.1 +chardet==3.0.4 +charset-normalizer==3.0.1 +click==7.1.2 +cliff==3.10.1 +cmd2==2.4.3 +colorama==0.4.5 +colorlog==4.0.2 +configparser==3.5.3 +coverage==5.3 +croniter==0.3.37 +cryptography==3.4.8 +debtcollector==2.5.0 +Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5 +decorator==4.4.2 +deepdiff==5.7.0 +defusedxml==0.7.1 +dill==0.2.9 +distlib==0.3.6 +dnspython==2.2.1 +docutils==0.17 +dogpile.cache==1.2.1 +drydock-provisioner @ git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#subdirectory=python +dulwich==0.20.50 +dumb-init==1.2.5.post1 +eventlet==0.33.3 +extras==1.0.0 +falcon==3.1.1 +fasteners==0.18 +filelock==3.4.1 +fixtures==3.0.0 +flake8==3.8.4 +Flask==1.1.0 +Flask-Admin==1.5.3 +Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6 +Flask-Babel==0.12.2 +Flask-Caching==1.3.3 +Flask-JWT-Extended==3.25.1 +Flask-Login==0.4.1 +Flask-OpenID==1.3.0 +Flask-SQLAlchemy==2.5.1 +flask-swagger==0.2.13 +Flask-WTF==0.14.3 +flower==0.9.7 +funcsigs==1.0.0 +future==0.16.0 +futurist==2.4.1 +gitdb==4.0.9 +GitPython==3.1.18 +google-auth==2.19.0 +greenlet==2.0.2 +gunicorn==19.10.0 +hacking==4.1.0 +hmsclient==0.1.1 +html5lib==0.9999999 +httpexceptor==1.4.0 +humanize==3.14.0 +idna==2.10 +imagesize==1.4.1 +importlib-metadata==2.1.3 +importlib-resources==5.4.0 +iso8601==1.1.0 +itsdangerous==2.0.1 +JayDeBeApi==1.2.3 +Jinja2==2.10.3 +JPype1==1.3.0 +json-merge-patch==0.2 +jsonpath-ng==1.5.3 +jsonpath-rw==1.4.0 +jsonpath-rw-ext==1.2.2 +jsonpickle==1.4.1 +jsonschema==3.2.0 +keystoneauth1==5.1.1 +keystonemiddleware==9.5.0 +kombu==4.6.3 +kubernetes==26.1.0 +lazy-object-proxy==1.7.1 +lockfile==0.12.2 +Mako==1.1.6 +Markdown==2.6.11 +MarkupSafe==2.0.1 +marshmallow==2.19.5 +marshmallow-enum==1.5.1 +marshmallow-sqlalchemy==0.18.0 +mccabe==0.6.1 +mock==5.0.2 +msgpack==1.0.5 +natsort==8.2.0 +netaddr==0.8.0 +netifaces==0.11.0 +networkx==2.5.1 +nose==1.3.7 +numpy==1.19.5 +oauthlib==3.2.2 +ordered-set==4.0.2 +ordereddict==1.1 +os-service-types==1.7.0 +oslo.cache==2.10.1 +oslo.concurrency==4.5.1 +oslo.config==8.7.1 +oslo.context==4.1.0 +oslo.db==10.0.0 +oslo.i18n==5.1.0 +oslo.log==4.6.0 +oslo.messaging==10.3.0 +oslo.middleware==4.4.0 +oslo.policy==3.10.1 +oslo.serialization==4.2.0 +oslo.service==2.8.0 +oslo.utils==4.12.3 +packaging==21.3 +pandas==0.25.3 +Paste==3.5.0 +PasteDeploy==2.1.1 +PasteScript==3.3.0 +pbr==5.5.1 +pendulum==1.4.4 +pip==21.3.1 +platformdirs==2.4.0 +ply==3.11 +prettytable==2.5.0 +prison==0.1.0 +promenade @ git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb +prometheus-client==0.8.0 +protobuf==3.19.6 +psutil==5.9.5 +psycopg2-binary==2.8.4 +py==1.11.0 +pyarrow==6.0.1 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pycadf==3.1.1 +pycodestyle==2.6.0 +pycparser==2.21 +pyflakes==2.2.0 +Pygments==2.14.0 +PyHive==0.6.5 +PyJWT==1.7.1 +pylibyaml==0.1.0 +pymongo==4.1.1 +pyparsing==2.4.7 +pyperclip==1.8.2 +pyproject-api==0.1.0 +pyrsistent==0.18.0 +python-barbicanclient==5.2.0 +python-daemon==2.1.2 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-keystoneclient==4.5.0 +python-memcached==1.59 +python-mimeparse==1.6.0 +python-subunit==1.4.0 +python3-openid==3.2.0 +pytz==2018.5 +pytzdata==2020.1 +PyYAML==5.4.1 +regex==2020.11.13 +reno==4.0.0 +repoze.lru==0.7 +requests==2.23.0 +requests-oauthlib==1.3.1 +resolver==0.2.1 +responses==0.12.1 +retry==0.9.2 +rfc3986==1.5.0 +Routes==2.5.1 +rsa==4.9 +selector==0.10.1 +setproctitle==1.2.3 +setuptools==45.2.0 +simplejson==3.19.1 +six==1.16.0 +smmap==5.0.0 +snakebite==2.11.0 +snowballstemmer==2.2.0 +Sphinx==3.3.1 +sphinx-rtd-theme==0.5.0 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +SQLAlchemy==1.3.15 +sqlalchemy-migrate==0.13.0 +sqlparse==0.4.4 +statsd==4.0.1 +stevedore==3.5.2 +tabulate==0.8.3 +Tempita==0.5.2 +tenacity==4.12.0 +termcolor==1.1.0 +testrepository==0.0.20 +testresources==2.0.1 +testscenarios==0.5.0 +testtools==2.5.0 +text-unidecode==1.2 +thrift==0.16.0 +tiddlyweb==2.4.3 +tomli==1.2.3 +tomlkit==0.11.6 +tornado==5.1.1 +typing-extensions==3.7.4.3 +tzlocal==1.5.1 +ulid==1.1 +unicodecsv==0.14.1 +urllib3==1.25.11 +uWSGI==2.0.21 +vine==1.3.0 +virtualenv==20.16.2 +wcwidth==0.2.6 +WebOb==1.8.7 +websocket-client==1.3.1 +Werkzeug==0.16.1 +wheel==0.37.1 +wrapt==1.15.0 +wsgi-intercept==1.11.0 +WTForms==2.3.3 +xmltodict==0.13.0 +yappi==1.4.0 +yq==3.2.1 +zipp==3.6.0 +zope.deprecation==4.4.0 diff --git a/src/bin/shipyard_airflow/setup.cfg b/src/bin/shipyard_airflow/setup.cfg index a934df2e..88fa9f6b 100644 --- a/src/bin/shipyard_airflow/setup.cfg +++ b/src/bin/shipyard_airflow/setup.cfg @@ -12,6 +12,7 @@ classifier = Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.10 diff --git a/src/bin/shipyard_airflow/tox.ini b/src/bin/shipyard_airflow/tox.ini index 8bc6ad26..1f1639ab 100644 --- a/src/bin/shipyard_airflow/tox.ini +++ b/src/bin/shipyard_airflow/tox.ini @@ -12,6 +12,17 @@ deps = -r{toxinidir}/test-requirements.txt +[testenv:freeze-bionic] +recreate = True +allowlist_externals= + rm + sh +deps= + -r{toxinidir}/requirements-direct.txt +commands= + rm -f {toxinidir}/requirements-frozen-bionic.txt + sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt" + [testenv:freeze] recreate = True allowlist_externals= @@ -28,11 +39,28 @@ commands= skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes +basepython=python3.8 allowlist_externals= bash airflow commands = - bash -c "rm -f ${HOME}/airflow/airflow.db" + bash -c "rm -f $HOME/airflow/airflow.db" + pip install -r{toxinidir}/test-requirements.txt + airflow initdb + airflow run example_bash_operator runme_0 2018-01-01 + airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 + airflow dag_state example_bash_operator 2018-01-01 + pytest {posargs} -vv + +[testenv:cover] +skipsdist=True +setenv= + SLUGIFY_USES_TEXT_UNIDECODE=yes +allowlist_externals= + bash + airflow +commands = + bash -c "rm -f $HOME/airflow/airflow.db" pip install -r{toxinidir}/test-requirements.txt airflow initdb airflow run example_bash_operator runme_0 2018-01-01 @@ -44,9 +72,31 @@ commands = --cov-report=term-missing:skip-covered \ --cov-config=.coveragerc \ --cov=shipyard_airflow \ - --cov-report=html \ + --cov-report html:cover \ + --cov-report xml:cover/coverage.xml \ + --cov-report term \ -vv +[testenv:py36] +skipsdist=True +basepython=python3.6 +deps = + -r{toxinidir}/requirements-frozen-bionic.txt +setenv= + SLUGIFY_USES_TEXT_UNIDECODE=yes +allowlist_externals= + bash + airflow +commands = + bash -c "rm -f $HOME/airflow/airflow.db" + pip install -r{toxinidir}/test-requirements.txt + airflow initdb + airflow run example_bash_operator runme_0 2018-01-01 + airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 + airflow dag_state example_bash_operator 2018-01-01 + pytest {posargs} -vv + + [testenv:pep8] deps= flake8>=3.3.0 diff --git a/src/bin/shipyard_client/requirements-frozen-bionic.txt b/src/bin/shipyard_client/requirements-frozen-bionic.txt new file mode 100644 index 00000000..597a2e7d --- /dev/null +++ b/src/bin/shipyard_client/requirements-frozen-bionic.txt @@ -0,0 +1,30 @@ +arrow==0.17.0 +certifi==2023.5.7 +chardet==3.0.4 +click==7.1.2 +click-default-group==1.2 +coverage==5.3 +Flask==1.1.0 +idna==2.10 +importlib-metadata==4.8.3 +iso8601==1.1.0 +itsdangerous==2.0.1 +Jinja2==2.10.3 +keystoneauth1==5.1.1 +MarkupSafe==2.0.1 +os-service-types==1.7.0 +pbr==5.5.1 +pip==21.3.1 +PTable==0.9.2 +pylibyaml==0.1.0 +python-dateutil==2.8.1 +PyYAML==5.4.1 +requests==2.23.0 +setuptools==45.2.0 +six==1.16.0 +stevedore==3.5.2 +typing_extensions==4.1.1 +urllib3==1.25.11 +Werkzeug==0.16.1 +wheel==0.37.1 +zipp==3.6.0 diff --git a/src/bin/shipyard_client/tox.ini b/src/bin/shipyard_client/tox.ini index 9cbde55a..46b45c88 100644 --- a/src/bin/shipyard_client/tox.ini +++ b/src/bin/shipyard_client/tox.ini @@ -7,8 +7,21 @@ setenv= PYTHONWARNING=all LC_ALL=C.UTF-8 LC_ALL=C.UTF-8 -deps = -r{toxinidir}/requirements-frozen.txt - -r{toxinidir}/test-requirements.txt +deps = + -r{toxinidir}/requirements-frozen.txt + -r{toxinidir}/test-requirements.txt + +[testenv:freeze-bionic] +recreate = True +allowlist_externals= + rm + sh +deps= + -r{toxinidir}/requirements-direct.txt +commands= + rm -f {toxinidir}/requirements-frozen-bionic.txt + sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt" + [testenv:freeze] recreate = True @@ -21,9 +34,25 @@ commands= rm -f {toxinidir}/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" +[testenv:py36] +skipsdist=True +deps = + -r{toxinidir}/requirements-frozen-bionic.txt + -r{toxinidir}/test-requirements.txt +commands = + pytest \ + {posargs} \ + -vv [testenv:py38] skipsdist=True +commands = + pytest \ + {posargs} \ + -vv + +[testenv:cover] +skipsdist=True commands = pytest \ {posargs} \ @@ -31,9 +60,12 @@ commands = --cov-report=term-missing:skip-covered \ --cov-config=.coveragerc \ --cov=shipyard_client \ - --cov-report=html \ + --cov-report html:cover \ + --cov-report xml:cover/coverage.xml \ + --cov-report term \ -vv + [testenv:pep8] deps= flake8>=3.3.0 diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml new file mode 100644 index 00000000..32ccc3b8 --- /dev/null +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -0,0 +1,101 @@ +- hosts: all + roles: + - bindep + - ensure-docker + - ensure-python + - ensure-pip + - ensure-tox + + tasks: + - include_vars: vars.yaml + + - name: Debug tag generation inputs + block: + - debug: + var: publish + - debug: + var: tags + - debug: + var: distro + - debug: + var: zuul + - debug: + msg: "{{ tags | to_json }}" + + - name: Determine tags + shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py + environment: + BRANCH: "{{ zuul.branch | default('') }}" + CHANGE: "{{ zuul.change | default('') }}" + COMMIT: "{{ zuul.newrev | default('') }}" + PATCHSET: "{{ zuul.patchset | default('') }}" + register: image_tags + + - name: Debug computed tags + debug: + var: image_tags + + - name: Install Docker python module for ansible docker login + block: + - pip: + name: docker + version: 4.4.4 + executable: pip3 + become: True + + - name: Make images - verbosive + when: not publish + shell: | + make run_images + args: + chdir: "{{ zuul.project.src_dir }}" + executable: /bin/bash + become: True + + + - name: Make images + when: not publish + block: + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + DISTRO: "{{ distro }}" + IMAGE_TAG: "{{ item }}" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + become: True + + - name: Publish images + block: + - docker_login: + username: "{{ airship_shipyard_quay_creds.username }}" + password: "{{ airship_shipyard_quay_creds.password }}" + registry_url: "https://quay.io/api/v1/" + + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + DOCKER_REGISTRY: "quay.io" + IMAGE_PREFIX: "airshipit" + DISTRO: "{{ distro }}" + IMAGE_TAG: "{{ item }}" + COMMIT: "{{ zuul.newrev | default('') }}" + PUSH_IMAGE: "true" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + when: publish + become: True \ No newline at end of file diff --git a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml index 63ec9353..c6f9cadd 100644 --- a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml +++ b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml @@ -47,6 +47,17 @@ become: True +- name: Make and run images - verbosive + when: not publish + shell: | + make run_images + args: + chdir: "{{ zuul.project.src_dir }}" + executable: /bin/bash + become: True + + + - name: Make images when: not publish block: diff --git a/tox.ini b/tox.ini index 50fc22f1..4a9942da 100644 --- a/tox.ini +++ b/tox.ini @@ -49,6 +49,27 @@ commands= bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample" bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample" +[testenv:py36] +deps = + tox<=4.0.0 +allowlist_externals= + tox +commands= + tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py36 + tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py36 + +[testenv:cover] +deps = + tox<=4.0.0 +allowlist_externals= + tox + sh +commands= + tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e cover + tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e cover + sh -c 'mv ./src/bin/shipyard_airflow/cover ./' + sh -c 'mv ./src/bin/shipyard_client/cover ./cover/' + [testenv:pep8] allowlist_externals= tox