airship/shipyard
Code Issues Proposed changes

Restored ubuntu_bionic image build

Browse Source 
This PS restores image build for ubuntu_bionic and adds appropriate
gates to keep it tested by appropriate functional and integrational
tests.

Change-Id: I4c4c7186e67c771f8249b2017b2507aac17760b7
This commit is contained in:
Sergiy Markin 2023-05-26 15:52:13 +00:00
parent 9c28c832dd
commit 7ea83c25ef
15 changed files with 608 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
.zuul.yaml
View File

@ -14,6 +14,7 @@
templates:
- docs-on-readthedocs
- openstack-python38-jobs
- openstack-python36-jobs
vars:
rtd_webhook_id: '38576'
rtd_project_name: 'airship-shipyard'
@ -21,20 +22,28 @@
check:
jobs:
- openstack-tox-pep8
- airship-shipyard-openstack-tox-cover-focal
- airship-shipyard-chart-build-gate
- airship-shipyard-chart-build-latest-htk
- airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment
- airship-shipyard-airskiff-deployment-bionic
- airship-shipyard-airskiff-deployment-focal
- airship-shipyard-image-gate-ubuntu_focal
- airship-shipyard-image-gate-ubuntu_bionic
gate:
jobs:
- openstack-tox-pep8
- airship-shipyard-openstack-tox-cover-focal
- airship-shipyard-chart-build-gate
- airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment-bionic
- airship-shipyard-airskiff-deployment-focal
- airship-shipyard-image-gate-ubuntu_focal
- airship-shipyard-image-gate-ubuntu_bionic
post:
jobs:
- airship-shipyard-docker-build-post-ubuntu_focal
- airship-shipyard-docker-build-publish-ubuntu_focal
- airship-shipyard-docker-build-publish-ubuntu_bionic
- shipyard-upload-git-mirror
- nodeset:
@ -49,12 +58,21 @@
- name: primary
label: ubuntu-focal
- job:
name: airship-shipyard-openstack-tox-cover-focal
parent: openstack-tox-cover
description: Runs cover job on focal
nodeset: airship-shipyard-single-node-focal
- job:
name: airship-shipyard-chart-build-gate
description: Build charts using pinned Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: airship-shipyard-single-node-focal
vars:
HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b
- job:
name: airship-shipyard-chart-build-latest-htk
@ -78,13 +96,14 @@
description: |
Base job for running airship-shipyard image related jobs.
timeout: 3600
run: tools/gate/playbooks/run-image.yaml
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: airship-shipyard-single-node-focal
irrelevant-files:
irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
- job:
name: airship-shipyard-image-gate-ubuntu_focal
description: |
@ -98,7 +117,20 @@
patch_set: true
- job:
name: airship-shipyard-airskiff-deployment
name: airship-shipyard-image-gate-ubuntu_bionic
description: |
Run shipyard-image build for ubuntu_bionic
parent: airship-shipyard-image-base
irrelevant-files: *irrelevant-files
vars:
publish: false
distro: ubuntu_bionic
tags:
dynamic:
patch_set: true
- job:
name: airship-shipyard-airskiff-deployment-focal
nodeset: airship-shipyard-single-node-focal
description: |
Deploy Memcached using Airskiff and submitted Armada changes.
@ -114,29 +146,35 @@
vars:
CLONE_SHIPYARD: false
HTK_COMMIT: master
irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
SHIPYARD_IMAGE_DISTRO: ubuntu_focal
irrelevant-files: *irrelevant-files
- job:
name: airship-shipyard-image-gate-opensuse
name: airship-shipyard-airskiff-deployment-bionic
nodeset: airship-shipyard-single-node-focal
description: |
Run shipyard-image build for opensuse.
parent: airship-shipyard-image-base
Deploy Memcached using Airskiff and submitted Armada changes.
timeout: 9600
voting: true
pre-run:
- tools/gate/playbooks/airskiff-reduce-site.yaml
- tools/gate/playbooks/git-config.yaml
run: tools/gate/playbooks/airskiff-deploy.yaml
post-run: tools/gate/playbooks/debug-report.yaml
required-projects:
- airship/treasuremap
vars:
publish: false
distro: opensuse_15
tags:
dynamic:
patch_set: true
CLONE_SHIPYARD: false
HTK_COMMIT: master
SHIPYARD_IMAGE_DISTRO: ubuntu_bionic
irrelevant-files: *irrelevant-files
- job:
name: airship-shipyard-docker-build-post-ubuntu_focal
name: airship-shipyard-docker-build-publish-ubuntu_focal
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node
nodeset: airship-shipyard-single-node-focal
irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
@ -151,19 +189,16 @@
- latest
- job:
name: airship-shipyard-docker-build-post-opensuse
name: airship-shipyard-docker-build-publish-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
nodeset: airship-shipyard-single-node-focal
irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
vars:
publish: true
distro: opensuse_15
distro: ubuntu_bionic
tags:
dynamic:
branch: true
@ -171,6 +206,7 @@
static:
- latest
- secret:
name: airship_shipyard_quay_creds
data:
@ -205,15 +241,15 @@
git_mirror_repository: airshipit/shipyard
secrets:
- name: git_mirror_credentials
secret: shipyard-airshipit-github-secret
secret: shipyard-airshipit-github-secret-2023-03-23
pass-to-parent: true
- secret:
name: shipyard-airshipit-github-secret
name: shipyard-airshipit-github-secret-2023-03-23
data:
user: git
host: github.com
host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
ssh_key: !encrypted/pkcs1-oaep
- hUsE3gOWqb6aoNVSpqRHcUbqi0wOjRJ47viZWr7JopICoyhIerZ6DN4dGRaFjJ+8UtQvD
uXKYHZ35+YbsJDQttIadtVvj5Va8hIToI/rARSktyDp4zb+fK829INKuOjSszowhGoDuv

38
Makefile
View File

@ -40,18 +40,36 @@ IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO}
IMAGE_DIR:=images/$(IMAGE_NAME)
.PHONY: images
#Build all images in the list
images: $(IMAGE_NAME)
#Build and run all images in list
#sudo make images IMAGE_NAME=airflow will Build and Run airflow
#sudo make images will build and run airflow and shipyard
$(IMAGE_NAME):
@echo
@echo "===== Processing [$@] image ====="
@make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/$@ IMAGE_NAME=$@
@make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/$@_image_run.sh
# Build all docker images for this project
images: build_images
build_images: build_airflow build_shipyard
run_images: build_airflow run_airflow build_shipyard run_shipyard
#Build all images in list
build_airflow:
@echo
@echo "===== Processing [airflow] image ====="
@make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/airflow:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/airflow IMAGE_NAME=airflow
build_shipyard:
@echo
@echo "===== Processing [shipyard] image ====="
@make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/shipyard:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/shipyard IMAGE_NAME=shipyard
#Run all images in list
run_airflow:
@echo
@echo "===== Processing [airflow] image ====="
@make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/airflow:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/airflow_image_run.sh
run_shipyard:
@echo
@echo "===== Processing [shipyard] image ====="
@make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/shipyard:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/shipyard_image_run.sh
# Create tgz of the chart
.PHONY: charts

9
images/airflow/Dockerfile.ubuntu_bionic
View File

@ -64,10 +64,13 @@ RUN set -ex && \
libffi-dev \
libssl-dev \
libpq-dev \
libpq5 \
libtool \
locales \
netcat \
netbase \
postgresql-client \
postgresql-common \
python3 \
python3-setuptools \
python3-pip \
@ -110,7 +113,7 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
# Note - removing snakebite (python 2 vs. 3). See:
# https://github.com/puckel/docker-airflow/issues/77
# Install Airflow directly to allow overriding source
COPY images/airflow/requirements.txt /tmp/
COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /tmp/requirements.txt
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
&& pip3 install $AIRFLOW_SRC --no-cache-dir \
&& (pip3 uninstall -y snakebite || true) \
@ -135,8 +138,8 @@ ENV PBR_VERSION 0.1a1
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
# COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /tmp/api_requirements.txt
# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \

4
images/airflow/Dockerfile.ubuntu_focal
View File

@ -148,8 +148,8 @@ ENV PBR_VERSION 0.1a1
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
# COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \

4
images/shipyard/Dockerfile.ubuntu_bionic
View File

@ -85,8 +85,8 @@ RUN chown -R shipyard: /home/shipyard \
&& chmod +x /home/shipyard/entrypoint.sh
# Requirements and Shipyard source
COPY ${ctx_base}/shipyard_airflow/requirements.txt /home/shipyard/api_requirements.txt
COPY ${ctx_base}/shipyard_client/requirements.txt /home/shipyard/client_requirements.txt
COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /home/shipyard/api_requirements.txt
COPY ${ctx_base}/shipyard_client/requirements-frozen-bionic.txt /home/shipyard/client_requirements.txt
COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/

8
src/bin/shipyard_airflow/requirements-direct.txt
View File

@ -34,7 +34,7 @@ WTForms<=2.3.3
itsdangerous==2.0.1
git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e#egg=deckhand
git+https://opendev.org/airship/drydock.git@375abedb8aa3a413ca70a47aef467f39d65e1aee#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97#egg=armada
git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c#egg=promenade
git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand
git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f#egg=armada
git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb#egg=promenade

241
src/bin/shipyard_airflow/requirements-frozen-bionic.txt Normal file
View File

@ -0,0 +1,241 @@
alabaster==0.7.13
alembic==1.0.1
amqp==2.6.1
apache-airflow==1.10.5
apispec==1.3.3
argcomplete==3.0.8
Armada @ git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f
arrow==0.17.0
attrs==22.2.0
autopage==0.5.1
Babel==2.11.0
bcrypt==4.0.1
Beaker==1.12.0
billiard==3.6.4.0
cached-property==1.5.2
cachetools==4.2.4
celery==4.3.0
certifi==2023.5.7
cffi==1.15.1
chardet==3.0.4
charset-normalizer==3.0.1
click==7.1.2
cliff==3.10.1
cmd2==2.4.3
colorama==0.4.5
colorlog==4.0.2
configparser==3.5.3
coverage==5.3
croniter==0.3.37
cryptography==3.4.8
debtcollector==2.5.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5
decorator==4.4.2
deepdiff==5.7.0
defusedxml==0.7.1
dill==0.2.9
distlib==0.3.6
dnspython==2.2.1
docutils==0.17
dogpile.cache==1.2.1
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#subdirectory=python
dulwich==0.20.50
dumb-init==1.2.5.post1
eventlet==0.33.3
extras==1.0.0
falcon==3.1.1
fasteners==0.18
filelock==3.4.1
fixtures==3.0.0
flake8==3.8.4
Flask==1.1.0
Flask-Admin==1.5.3
Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6
Flask-Babel==0.12.2
Flask-Caching==1.3.3
Flask-JWT-Extended==3.25.1
Flask-Login==0.4.1
Flask-OpenID==1.3.0
Flask-SQLAlchemy==2.5.1
flask-swagger==0.2.13
Flask-WTF==0.14.3
flower==0.9.7
funcsigs==1.0.0
future==0.16.0
futurist==2.4.1
gitdb==4.0.9
GitPython==3.1.18
google-auth==2.19.0
greenlet==2.0.2
gunicorn==19.10.0
hacking==4.1.0
hmsclient==0.1.1
html5lib==0.9999999
httpexceptor==1.4.0
humanize==3.14.0
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
importlib-resources==5.4.0
iso8601==1.1.0
itsdangerous==2.0.1
JayDeBeApi==1.2.3
Jinja2==2.10.3
JPype1==1.3.0
json-merge-patch==0.2
jsonpath-ng==1.5.3
jsonpath-rw==1.4.0
jsonpath-rw-ext==1.2.2
jsonpickle==1.4.1
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==9.5.0
kombu==4.6.3
kubernetes==26.1.0
lazy-object-proxy==1.7.1
lockfile==0.12.2
Mako==1.1.6
Markdown==2.6.11
MarkupSafe==2.0.1
marshmallow==2.19.5
marshmallow-enum==1.5.1
marshmallow-sqlalchemy==0.18.0
mccabe==0.6.1
mock==5.0.2
msgpack==1.0.5
natsort==8.2.0
netaddr==0.8.0
netifaces==0.11.0
networkx==2.5.1
nose==1.3.7
numpy==1.19.5
oauthlib==3.2.2
ordered-set==4.0.2
ordereddict==1.1
os-service-types==1.7.0
oslo.cache==2.10.1
oslo.concurrency==4.5.1
oslo.config==8.7.1
oslo.context==4.1.0
oslo.db==10.0.0
oslo.i18n==5.1.0
oslo.log==4.6.0
oslo.messaging==10.3.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.2.0
oslo.service==2.8.0
oslo.utils==4.12.3
packaging==21.3
pandas==0.25.3
Paste==3.5.0
PasteDeploy==2.1.1
PasteScript==3.3.0
pbr==5.5.1
pendulum==1.4.4
pip==21.3.1
platformdirs==2.4.0
ply==3.11
prettytable==2.5.0
prison==0.1.0
promenade @ git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb
prometheus-client==0.8.0
protobuf==3.19.6
psutil==5.9.5
psycopg2-binary==2.8.4
py==1.11.0
pyarrow==6.0.1
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycadf==3.1.1
pycodestyle==2.6.0
pycparser==2.21
pyflakes==2.2.0
Pygments==2.14.0
PyHive==0.6.5
PyJWT==1.7.1
pylibyaml==0.1.0
pymongo==4.1.1
pyparsing==2.4.7
pyperclip==1.8.2
pyproject-api==0.1.0
pyrsistent==0.18.0
python-barbicanclient==5.2.0
python-daemon==2.1.2
python-dateutil==2.8.1
python-editor==1.0.4
python-keystoneclient==4.5.0
python-memcached==1.59
python-mimeparse==1.6.0
python-subunit==1.4.0
python3-openid==3.2.0
pytz==2018.5
pytzdata==2020.1
PyYAML==5.4.1
regex==2020.11.13
reno==4.0.0
repoze.lru==0.7
requests==2.23.0
requests-oauthlib==1.3.1
resolver==0.2.1
responses==0.12.1
retry==0.9.2
rfc3986==1.5.0
Routes==2.5.1
rsa==4.9
selector==0.10.1
setproctitle==1.2.3
setuptools==45.2.0
simplejson==3.19.1
six==1.16.0
smmap==5.0.0
snakebite==2.11.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==1.3.15
sqlalchemy-migrate==0.13.0
sqlparse==0.4.4
statsd==4.0.1
stevedore==3.5.2
tabulate==0.8.3
Tempita==0.5.2
tenacity==4.12.0
termcolor==1.1.0
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
text-unidecode==1.2
thrift==0.16.0
tiddlyweb==2.4.3
tomli==1.2.3
tomlkit==0.11.6
tornado==5.1.1
typing-extensions==3.7.4.3
tzlocal==1.5.1
ulid==1.1
unicodecsv==0.14.1
urllib3==1.25.11
uWSGI==2.0.21
vine==1.3.0
virtualenv==20.16.2
wcwidth==0.2.6
WebOb==1.8.7
websocket-client==1.3.1
Werkzeug==0.16.1
wheel==0.37.1
wrapt==1.15.0
wsgi-intercept==1.11.0
WTForms==2.3.3
xmltodict==0.13.0
yappi==1.4.0
yq==3.2.1
zipp==3.6.0
zope.deprecation==4.4.0

1
src/bin/shipyard_airflow/setup.cfg
View File

@ -12,6 +12,7 @@ classifier =
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.10

54
src/bin/shipyard_airflow/tox.ini
View File

@ -12,6 +12,17 @@ deps =
-r{toxinidir}/test-requirements.txt
[testenv:freeze-bionic]
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f {toxinidir}/requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
[testenv:freeze]
recreate = True
allowlist_externals=
@ -28,11 +39,28 @@ commands=
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
basepython=python3.8
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f ${HOME}/airflow/airflow.db"
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
pytest {posargs} -vv
[testenv:cover]
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
@ -44,9 +72,31 @@ commands =
--cov-report=term-missing:skip-covered \
--cov-config=.coveragerc \
--cov=shipyard_airflow \
--cov-report=html \
--cov-report html:cover \
--cov-report xml:cover/coverage.xml \
--cov-report term \
-vv
[testenv:py36]
skipsdist=True
basepython=python3.6
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
pytest {posargs} -vv
[testenv:pep8]
deps=
flake8>=3.3.0

30
src/bin/shipyard_client/requirements-frozen-bionic.txt Normal file
View File

@ -0,0 +1,30 @@
arrow==0.17.0
certifi==2023.5.7
chardet==3.0.4
click==7.1.2
click-default-group==1.2
coverage==5.3
Flask==1.1.0
idna==2.10
importlib-metadata==4.8.3
iso8601==1.1.0
itsdangerous==2.0.1
Jinja2==2.10.3
keystoneauth1==5.1.1
MarkupSafe==2.0.1
os-service-types==1.7.0
pbr==5.5.1
pip==21.3.1
PTable==0.9.2
pylibyaml==0.1.0
python-dateutil==2.8.1
PyYAML==5.4.1
requests==2.23.0
setuptools==45.2.0
six==1.16.0
stevedore==3.5.2
typing_extensions==4.1.1
urllib3==1.25.11
Werkzeug==0.16.1
wheel==0.37.1
zipp==3.6.0

38
src/bin/shipyard_client/tox.ini
View File

@ -7,8 +7,21 @@ setenv=
PYTHONWARNING=all
LC_ALL=C.UTF-8
LC_ALL=C.UTF-8
deps = -r{toxinidir}/requirements-frozen.txt
-r{toxinidir}/test-requirements.txt
deps =
-r{toxinidir}/requirements-frozen.txt
-r{toxinidir}/test-requirements.txt
[testenv:freeze-bionic]
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f {toxinidir}/requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
[testenv:freeze]
recreate = True
@ -21,9 +34,25 @@ commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
[testenv:py36]
skipsdist=True
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
commands =
pytest \
{posargs} \
-vv
[testenv:py38]
skipsdist=True
commands =
pytest \
{posargs} \
-vv
[testenv:cover]
skipsdist=True
commands =
pytest \
{posargs} \
@ -31,9 +60,12 @@ commands =
--cov-report=term-missing:skip-covered \
--cov-config=.coveragerc \
--cov=shipyard_client \
--cov-report=html \
--cov-report html:cover \
--cov-report xml:cover/coverage.xml \
--cov-report term \
-vv
[testenv:pep8]
deps=
flake8>=3.3.0

16
tools/gate/playbooks/airskiff-deploy.yaml
View File

@ -45,13 +45,15 @@
- name: Build Shipyard and Airflow with submitted changes
shell: |
make
docker pull quay.io/airshipit/shipyard:latest-ubuntu_bionic
docker rm quay.io/airshipit/shipyard:latest-ubuntu_bionic
docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal quay.io/airshipit/shipyard:latest-ubuntu_bionic
docker pull quay.io/airshipit/airflow:latest-ubuntu_bionic
docker rm quay.io/airshipit/airflow:latest-ubuntu_bionic
docker tag quay.io/airshipit/airflow:latest-ubuntu_focal quay.io/airshipit/airflow:latest-ubuntu_bionic
set -ex
export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
make images
if test "${DISTRO}" = 'ubuntu_bionic'
then
# this trick is needed to use bionic image instead of focal in airskiff deployment test
docker tag quay.io/airshipit/airflow:latest-ubuntu_bionic quay.io/airshipit/airflow:latest-ubuntu_focal
docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic quay.io/airshipit/shipyard:latest-ubuntu_focal
fi
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes

101
tools/gate/playbooks/docker-image-build.yaml Normal file
View File

@ -0,0 +1,101 @@
- hosts: all
roles:
- bindep
- ensure-docker
- ensure-python
- ensure-pip
- ensure-tox
tasks:
- include_vars: vars.yaml
- name: Debug tag generation inputs
block:
- debug:
var: publish
- debug:
var: tags
- debug:
var: distro
- debug:
var: zuul
- debug:
msg: "{{ tags | to_json }}"
- name: Determine tags
shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py
environment:
BRANCH: "{{ zuul.branch | default('') }}"
CHANGE: "{{ zuul.change | default('') }}"
COMMIT: "{{ zuul.newrev | default('') }}"
PATCHSET: "{{ zuul.patchset | default('') }}"
register: image_tags
- name: Debug computed tags
debug:
var: image_tags
- name: Install Docker python module for ansible docker login
block:
- pip:
name: docker
version: 4.4.4
executable: pip3
become: True
- name: Make images - verbosive
when: not publish
shell: |
make run_images
args:
chdir: "{{ zuul.project.src_dir }}"
executable: /bin/bash
become: True
- name: Make images
when: not publish
block:
- make:
chdir: "{{ zuul.project.src_dir }}"
target: images
params:
DISTRO: "{{ distro }}"
IMAGE_TAG: "{{ item }}"
with_items: "{{ image_tags.stdout_lines }}"
- shell: "docker images"
register: docker_images
- debug:
var: docker_images
become: True
- name: Publish images
block:
- docker_login:
username: "{{ airship_shipyard_quay_creds.username }}"
password: "{{ airship_shipyard_quay_creds.password }}"
registry_url: "https://quay.io/api/v1/"
- make:
chdir: "{{ zuul.project.src_dir }}"
target: images
params:
DOCKER_REGISTRY: "quay.io"
IMAGE_PREFIX: "airshipit"
DISTRO: "{{ distro }}"
IMAGE_TAG: "{{ item }}"
COMMIT: "{{ zuul.newrev | default('') }}"
PUSH_IMAGE: "true"
with_items: "{{ image_tags.stdout_lines }}"
- shell: "docker images"
register: docker_images
- debug:
var: docker_images
when: publish
become: True

11
tools/gate/roles/build-images/tasks/airship-shipyard.yaml
View File

@ -47,6 +47,17 @@
become: True
- name: Make and run images - verbosive
when: not publish
shell: |
make run_images
args:
chdir: "{{ zuul.project.src_dir }}"
executable: /bin/bash
become: True
- name: Make images
when: not publish
block:

21
tox.ini
View File

@ -49,6 +49,27 @@ commands=
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
[testenv:py36]
deps =
tox<=4.0.0
allowlist_externals=
tox
commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py36
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py36
[testenv:cover]
deps =
tox<=4.0.0
allowlist_externals=
tox
sh
commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e cover
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e cover
sh -c 'mv ./src/bin/shipyard_airflow/cover ./'
sh -c 'mv ./src/bin/shipyard_client/cover ./cover/'
[testenv:pep8]
allowlist_externals=
tox