airship/deckhand
Code Issues Proposed changes

Restored ubuntu_bionic image build

Browse Source 
This PS restores image build for ubuntu_bionic and adds appropriate
gates to keep it tested by appropriate functional and integrational
tests.

Also the latest osh-infra commit was used with kubernetes 1.27.1 for
integration tests.

Change-Id: Ia2c951f27b96774b553e7c0c7c6809172312f753
This commit is contained in:
Sergiy Markin
2023-05-20 01:21:36 +00:00
parent 3a06b1b604
commit aa603805f8
13 changed files with 442 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
.zuul.yaml
View File

@@ -15,37 +15,54 @@
- openstack-cover-jobs
- docs-on-readthedocs
- openstack-python38-jobs
- openstack-python36-jobs
vars:
rtd_webhook_id: '38572'
rtd_project_name: 'airship-deckhand'
check:
jobs:
- deckhand-tox-py36-postgresql
- deckhand-tox-py38-postgresql
- deckhand-functional-uwsgi-py36
- deckhand-functional-uwsgi-py38
- deckhand-functional-docker-py38-ubuntu_focal
- deckhand-functional-docker-py36-ubuntu_bionic
- deckhand-integration-uwsgi-py36
- deckhand-integration-uwsgi-py38
- deckhand-integration-docker-py38-ubuntu_focal
- deckhand-integration-docker-py36-ubuntu_bionic
- deckhand-chart-build-gate
- deckhand-chart-build-latest-htk
- deckhand-docker-build-gate-ubuntu_focal
- deckhand-docker-build-gate-ubuntu_bionic
- openstack-tox-pep8
# non-voting, failing gate - has to be fixed in treasuremap
- deckhand-airskiff-deployment
- deckhand-airskiff-deployment-focal
- deckhand-airskiff-deployment-bionic
gate:
jobs:
- deckhand-tox-py36-postgresql
- deckhand-tox-py38-postgresql
- deckhand-functional-uwsgi-py36
- deckhand-functional-uwsgi-py38
- deckhand-functional-docker-py38-ubuntu_focal
- deckhand-functional-docker-py36-ubuntu_bionic
- deckhand-integration-uwsgi-py36
- deckhand-integration-uwsgi-py38
- deckhand-integration-docker-py38-ubuntu_focal
- deckhand-integration-docker-py36-ubuntu_bionic
- deckhand-chart-build-gate
- deckhand-docker-build-gate-ubuntu_focal
- deckhand-docker-build-gate-ubuntu_bionic
- openstack-tox-pep8
- deckhand-airskiff-deployment-focal
- deckhand-airskiff-deployment-bionic
post:
jobs:
- deckhand-upload-git-mirror
- deckhand-docker-publish-ubuntu_focal
- deckhand-docker-tag-ubuntu_focal
- deckhand-docker-publish-ubuntu_bionic
- deckhand-docker-tag-ubuntu_bionic
- nodeset:
@@ -81,6 +98,36 @@
vars:
tox_envlist: py38-postgresql
- job:
name: deckhand-tox-py36-postgresql
parent: openstack-tox-py38
nodeset: deckhand-single-node
pre-run:
- tools/gate/playbooks/install-postgresql.yaml
vars:
tox_envlist: py36-postgresql
- job:
name: deckhand-functional-uwsgi-py36
voting: true
description: |
Run tox-based functional tests for the Airship Deckhand project using a
minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
for ephemeral PostgreSQL DB, under cPython version 3.6.
run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml
nodeset: deckhand-single-node
vars:
tox_envlist: py38-functional-dev
distro: ubuntu_bionic
irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
- ^etc/.*$
- ^releasenotes/.*$
- ^setup.cfg$
- ^deckhand/tests/unit/.*$
- job:
name: deckhand-functional-uwsgi-py38
voting: true
@@ -91,7 +138,8 @@
run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml
nodeset: deckhand-single-node-focal
vars:
tox_envlist: functional-dev
tox_envlist: py38-functional-dev
distro: ubuntu_focal
irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
@@ -125,11 +173,41 @@
parent: deckhand-functional-docker-base
nodeset: deckhand-single-node-focal
vars:
tox_envlist: functional
tox_envlist: py38-functional
disable_keystone: true
distro: ubuntu_focal
irrelevant-files: *irrelevant-files
- job:
name: deckhand-functional-docker-py36-ubuntu_bionic
description: |
Run tox-based functional tests for the Airship Deckhand project under
cPython version 3.6. Uses tox with the ``functional-py36`` environment.
Ubuntu (focal) image is built and used.
parent: deckhand-functional-docker-base
nodeset: deckhand-single-node
vars:
tox_envlist: py36-functional
disable_keystone: true
distro: ubuntu_bionic
irrelevant-files: *irrelevant-files
- job:
name: deckhand-integration-uwsgi-py36
voting: true
description: |
Run tox-based integration tests for the Airship Deckhand project using a
minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
for ephemeral PostgreSQL DB, under cPython version 3.6.
timeout: 3600
run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml
nodeset: deckhand-single-node
irrelevant-files: *irrelevant-files
vars:
disable_keystone: true
distro: ubuntu_bionic
- job:
name: deckhand-integration-uwsgi-py38
@@ -144,6 +222,7 @@
irrelevant-files: *irrelevant-files
vars:
disable_keystone: true
distro: ubuntu_focal
- job:
name: deckhand-chart-build-gate
@@ -200,9 +279,19 @@
disable_keystone: false
distro: ubuntu_focal
- job:
name: deckhand-integration-docker-py36-ubuntu_bionic
description: |
Run tox-based integration tests for the Airship Deckhand project under
cPython version 3.6. Builds ubuntu (focal) deckhand image.
parent: deckhand-integration-docker-base
nodeset: openstack-helm-single-node
vars:
disable_keystone: false
distro: ubuntu_focal
- job:
name: deckhand-airskiff-deployment
name: deckhand-airskiff-deployment-focal
voting: true
nodeset: deckhand-single-node-airskiff-focal
description: |
@@ -217,12 +306,39 @@
- airship/treasuremap
vars:
CLONE_DECKHAND: false
OSH_INFRA_COMMIT: 97ce6d7d8e9a090c748800d69a57bbd9af698b60
OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
DECKHAND_IMAGE_DISTRO: ubuntu_focal
distro: ubuntu_focal
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
- job:
name: deckhand-airskiff-deployment-bionic
voting: true
nodeset: deckhand-single-node-airskiff-focal
description: |
Deploy Memcached using Airskiff and submitted Deckhand changes.
timeout: 9600
pre-run:
- tools/gate/playbooks/airskiff-reduce-site.yaml
- tools/gate/playbooks/git-config.yaml
run: tools/gate/playbooks/airskiff-deploy.yaml
post-run: tools/gate/playbooks/debug-report.yaml
required-projects:
- airship/treasuremap
vars:
CLONE_DECKHAND: false
OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
DECKHAND_IMAGE_DISTRO: ubuntu_bionic
distro: ubuntu_focal
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
- job:
name: deckhand-docker-build-gate-ubuntu_focal
timeout: 1800
@@ -242,6 +358,19 @@
dynamic:
patch_set: true
- job:
name: deckhand-docker-build-gate-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: deckhand-single-node
irrelevant-files: *non-code-files-template
vars:
publish: false
distro: ubuntu_bionic
tags:
dynamic:
patch_set: true
- job:
name: deckhand-docker-publish-ubuntu_focal
description: |
@@ -264,6 +393,27 @@
static:
- latest
- job:
name: deckhand-docker-publish-ubuntu_bionic
description: |
Runs on every merge, unless files in a dictionary below are changed.
Builds and publishes container ubuntu images on quay.io with a set of tags
listed in vars section. Waits in Zuul queue for a node (VM) assignment.
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: deckhand-single-node
secrets:
- airship_deckhand_quay_creds
irrelevant-files: *non-code-files-template
vars:
publish: true
distro: ubuntu_bionic
tags:
dynamic:
branch: true
commit: true
static:
- latest
- job:
name: deckhand-docker-tag-ubuntu_focal
@@ -281,6 +431,23 @@
vars:
distro: ubuntu_focal
- job:
name: deckhand-docker-tag-ubuntu_bionic
description: |
Runs on every merge when files in a dictionalry below are changed, and
adds git commit id tag onto the ubuntu container image published on quay.io,
which has `latest` tag set. Does not wait in queue for a node (VM)
assignment, runs almost immediately.
timeout: 1800
run: tools/gate/playbooks/docker-image-tag.yaml
nodeset:
nodes: []
secrets:
- airship_deckhand_quay_creds
vars:
distro: ubuntu_bionic
- secret:
name: airship_deckhand_quay_creds
@@ -328,15 +495,15 @@
git_mirror_repository: airshipit/deckhand
secrets:
- name: git_mirror_credentials
secret: deckhand-airshipit-github-secret
secret: deckhand-airshipit-github-secret-2023-03-23
pass-to-parent: true
- secret:
name: deckhand-airshipit-github-secret
name: deckhand-airshipit-github-secret-2023-03-23
data:
user: git
host: github.com
host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
ssh_key: !encrypted/pkcs1-oaep
- Lrtj2vWxjm4C2DEst6lxc5WIy0fnKP8U8miCMy9fhWJ27ai1WHde2CBbIibLQodx26cmj
VToF5DDIVyfteYzyQJ5pLK6myXR0tNbBwvskIardGFzOBZ0xxPDHAJa4qfIt8empPx1k8

10
images/deckhand/Dockerfile.ubuntu_bionic
View File

@@ -88,8 +88,14 @@ RUN chown -R deckhand: /home/deckhand \
# Set work directory and install dependencies
WORKDIR /home/deckhand
RUN pip3 install -r requirements.txt
RUN python3 setup.py install
RUN pip3 install --no-cache-dir -r requirements-frozen-bionic.txt
# Setting deckhand version for BPR
ENV PBR_VERSION 1.1
# RUN python3 setup.py install --verbose
RUN pip3 install -e . --verbose --use-pep517 \
&& echo "/home/deckhand" \
> /usr/local/lib/python3.6/dist-packages/deckhand.pth
# Set user to deckhand
USER deckhand

5
images/deckhand/Dockerfile.ubuntu_focal
View File

@@ -91,13 +91,12 @@ WORKDIR /home/deckhand
RUN pip3 install --no-cache-dir -r requirements-frozen.txt
# Setting deckhand version for BPR
ENV PBR_VERSION 1.0
ENV PBR_VERSION 1.1
COPY . /opt/deckhand
# RUN python3 setup.py install --verbose
RUN pip3 install -e . --verbose --use-pep517 \
&& echo "/opt/deckhand" \
&& echo "/home/deckhand" \
> /usr/local/lib/python3.8/dist-packages/deckhand.pth
# Set user to deckhand

2
requirements-direct.txt
View File

@@ -67,7 +67,7 @@ testresources
testscenarios
testtools<=2.5.0
tiddlyweb
typing-extensions==3.7.2
typing-extensions==3.7.4.3
urllib3 >= 1.21.1, <= 1.25.11
uWSGI==2.0.21
virtualenv

156
requirements-frozen-bionic.txt Normal file
View File

@@ -0,0 +1,156 @@
alabaster==0.7.13
alembic==1.4.3
amqp==2.6.1
argcomplete==3.0.8
attrs==22.2.0
autopage==0.5.1
Babel==2.11.0
bcrypt==4.0.1
Beaker==1.12.0
cachetools==4.2.4
certifi==2023.5.7
cffi==1.15.1
chardet==3.0.4
charset-normalizer==3.0.1
cliff==3.10.1
cmd2==2.4.3
configparser==5.2.0
coverage==6.2
cryptography==3.4.8
dataclasses==0.8
debtcollector==2.5.0
decorator==4.4.2
deepdiff==5.7.0
distlib==0.3.6
dnspython==2.2.1
docutils==0.17
dogpile.cache==1.2.0
dulwich==0.20.50
eventlet==0.33.3
extras==1.0.0
falcon==3.1.1
fasteners==0.18
filelock==3.4.1
fixtures==3.0.0
flake8==3.8.4
futurist==2.4.1
greenlet==2.0.2
hacking==4.1.0
html5lib==0.9999999
httpexceptor==1.4.0
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
importlib-resources==5.4.0
iso8601==1.1.0
Jinja2==3.0.3
jsonpath-ng==1.5.3
jsonpath-rw==1.4.0
jsonpath-rw-ext==1.2.2
jsonpickle==1.4.1
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==9.5.0
kombu==4.6.11
Mako==1.1.6
MarkupSafe==2.0.1
mccabe==0.6.1
msgpack==1.0.5
netaddr==0.8.0
netifaces==0.11.0
networkx==2.5.1
nose==1.3.7
ordered-set==4.0.2
os-service-types==1.7.0
oslo.cache==2.10.1
oslo.concurrency==4.5.1
oslo.config==8.7.1
oslo.context==4.1.0
oslo.db==10.0.0
oslo.i18n==5.1.0
oslo.log==4.6.0
oslo.messaging==12.13.0
oslo.metrics==0.4.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.2.0
oslo.service==2.8.0
oslo.utils==4.12.3
packaging==21.3
Paste==3.5.0
PasteDeploy==2.1.1
PasteScript==3.3.0
pbr==5.5.1
pip==21.3.1
platformdirs==2.4.0
ply==3.11
prettytable==2.5.0
prometheus-client==0.16.0
psycopg2-binary==2.9.6
pycadf==3.1.1
pycodestyle==2.6.0
pycparser==2.21
pyflakes==2.2.0
Pygments==2.14.0
pylibyaml==0.1.0
pymongo==4.1.1
pyparsing==2.4.7
pyperclip==1.8.2
pyproject-api==0.1.0
pyrsistent==0.18.0
python-barbicanclient==5.2.0
python-dateutil==2.8.1
python-editor==1.0.4
python-keystoneclient==4.5.0
python-memcached==1.59
python-mimeparse==1.6.0
python-subunit==1.4.0
pytz==2023.3
PyYAML==5.4.1
reno==4.0.0
repoze.lru==0.7
requests==2.23.0
resolver==0.2.1
rfc3986==1.5.0
Routes==2.5.1
selector==0.10.1
setuptools==45.2.0
simplejson==3.19.1
six==1.16.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==1.3.20
sqlalchemy-migrate==0.13.0
sqlparse==0.4.4
statsd==4.0.1
stevedore==3.5.2
Tempita==0.5.2
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
tiddlyweb==2.4.3
tomli==1.2.3
tomlkit==0.11.6
typing-extensions==3.7.4.3
urllib3==1.25.11
uWSGI==2.0.21
vine==1.3.0
virtualenv==20.16.2
wcwidth==0.2.6
WebOb==1.8.7
Werkzeug==2.0.3
wheel==0.37.1
wrapt==1.15.0
wsgi-intercept==1.11.0
xmltodict==0.13.0
yappi==1.4.0
yq==3.2.1
zipp==3.6.0

2
requirements-frozen.txt
View File

@@ -137,7 +137,7 @@ testtools==2.5.0
tiddlyweb==2.4.3
tomli==2.0.1
tomlkit==0.11.7
typing-extensions==3.7.2
typing-extensions==3.7.4.3
urllib3==1.25.11
uWSGI==2.0.21
vine==1.3.0

3
setup.cfg
View File

@@ -14,12 +14,13 @@ classifier =
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.10
[options]
include_package_data = True
python_requires = >=3.8
python_requires = >=3.6
[files]

9
tools/gate/playbooks/airskiff-deploy.yaml
View File

@@ -43,7 +43,14 @@
- name: Build Deckhand with submitted changes
shell: |
make
set -ex
export DISTRO={{ DECKHAND_IMAGE_DISTRO }}
make images
if test "${DISTRO}" = 'ubuntu_bionic'
then
# this trick is needed to use bionic image instead of focal in airskiff deployment test
docker tag quay.io/airshipit/deckhand:latest-ubuntu_bionic quay.io/airshipit/deckhand:latest-ubuntu_focal
fi
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes

5
tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml
View File

@@ -28,7 +28,12 @@
python3-dev
pip3 install --upgrade pip
pip3 install -r test-requirements.txt
if test "{{ distro }}" = 'ubuntu_bionic'
then
pip3 install -r requirements-frozen-bionic.txt
else
pip3 install -r requirements-frozen.txt
fi
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes

5
tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml
View File

@@ -56,7 +56,12 @@
# with OpenStack-Helm, which is used to orchestrate various OpenStack
# services in the integration script called below.
sudo -H -E pip3 install -e . --verbose --use-pep517
if test "{{ distro }}" = 'ubuntu_bionic'
then
sudo -E -H pip3 install -r requirements-frozen-bionic.txt
else
sudo -E -H pip3 install -r requirements-frozen.txt
fi
pifpaf run postgresql -- ./tools/integration-tests.sh
args:
chdir: "{{ zuul.project.src_dir }}"

2
tools/helm_install.sh
View File

@@ -17,7 +17,7 @@
set -x
HELM=$1
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz"}
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.11.1-linux-amd64.tar.gz"}
function install_helm_binary {

76
tox.ini
View File

@@ -58,6 +58,31 @@ commands=
sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"
[testenv:freeze-bionic]
basepython=python3
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt"
[testenv:py36]
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
allowlist_externals =
stestr
find
rm
commands =
{[testenv]commands}
stestr run {posargs}
stestr slowest
[testenv:py38]
allowlist_externals =
@@ -78,7 +103,34 @@ allowlist_externals =
find
rm
[testenv:functional]
[testenv:py36-postgresql]
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
commands =
{[testenv]commands}
bash {toxinidir}/tools/run_pifpaf.sh '{posargs}'
allowlist_externals =
bash
find
rm
[testenv:py36-functional]
basepython=python3
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
allowlist_externals =
find
stestr
pip3
echo
setenv = VIRTUAL_ENV={envdir}
commands =
find . -type f -name "*.pyc" -delete
stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs}
[testenv:py38-functional]
basepython=python3
allowlist_externals =
find
@@ -90,7 +142,8 @@ commands =
find . -type f -name "*.pyc" -delete
stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs}
[testenv:functional-dev]
[testenv:py38-functional-dev]
basepython=python3
# Minimalistic functional test job for running Deckhand functional tests
# via uwsgi. Uses pifpaf for DB instantiation. Useful for developers.
@@ -106,6 +159,25 @@ commands =
find . -type f -name "*.pyc" -delete
pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}"
[testenv:py36-functional-dev]
basepython=python3
# Minimalistic functional test job for running Deckhand functional tests
# via uwsgi. Uses pifpaf for DB instantiation. Useful for developers.
# Requires PostgreSQL be installed on host.
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
allowlist_externals =
find
stestr
pifpaf
pip3
setenv = VIRTUAL_ENV={envdir}
commands =
pip3 install -e . --verbose --use-pep517
find . -type f -name "*.pyc" -delete
pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}"
[testenv:cover]
allowlist_externals =
find