diff --git a/.zuul.yaml b/.zuul.yaml index e0b8eba0..20ea67ce 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -12,10 +12,10 @@ - project: templates: - - docs-on-readthedocs + # commented out because openstack-tox-docs fails on jammy zuul nodes + # - docs-on-readthedocs - openstack-python38-jobs - - openstack-python36-jobs - - openstack-cover-jobs + # - openstack-cover-jobs # - release-notes-jobs-python3 vars: rtd_webhook_id: '38574' @@ -34,17 +34,21 @@ check: jobs: - openstack-tox-pep8-focal - # - pegleg-dependency-vulnerability-check + - openstack-tox-cover-focal + - openstack-tox-docs-focal + - pegleg-dependency-vulnerability-check + - airship-pegleg-validate-airskiff-manifests-focal - airship-pegleg-docker-build-gate-ubuntu_focal - - airship-pegleg-docker-build-gate-ubuntu_bionic - airship-pegleg-lint-yaml gate: jobs: - openstack-tox-pep8-focal + - openstack-tox-cover-focal + - openstack-tox-docs-focal - pegleg-dependency-vulnerability-check + - airship-pegleg-validate-airskiff-manifests-focal - airship-pegleg-docker-build-gate-ubuntu_focal - - airship-pegleg-docker-build-gate-ubuntu_bionic - airship-pegleg-lint-yaml post: jobs: @@ -69,6 +73,17 @@ parent: openstack-tox-pep8 nodeset: openstack-single-node-focal +- job: + name: openstack-tox-docs-focal + parent: openstack-tox-docs + nodeset: openstack-single-node-focal + +- job: + name: openstack-tox-cover-focal + parent: openstack-tox-cover + nodeset: openstack-single-node-focal + + - job: name: airship-pegleg-lint-yaml voting: true @@ -79,19 +94,43 @@ - ^.*\.rst$ - ^doc/.*$ -# - job: -# name: pegleg-dependency-vulnerability-check -# parent: tox-py38 -# voting: false -# timeout: 600 -# nodeset: ubuntu-focal -# vars: -# tox_envlist: safety -# bindep_profile: test py38 +- job: + name: pegleg-dependency-vulnerability-check + parent: tox-py38 + voting: false + timeout: 600 + nodeset: ubuntu-focal + vars: + tox_envlist: safety + bindep_profile: test py38 + +- job: + name: airship-pegleg-validate-airskiff-manifests-focal + nodeset: airship-pegleg-single-node-focal + description: | + Test pegleg Airskiff site manifest validation + timeout: 9600 + voting: true + pre-run: + - tools/gate/playbooks/airskiff-reduce-site.yaml + - tools/gate/playbooks/git-config.yaml + run: tools/gate/playbooks/airskiff-validate.yaml + post-run: tools/gate/playbooks/debug-report.yaml + required-projects: + - airship/treasuremap + vars: + CLONE_PEGLEG: false + HTK_COMMIT: 97ce6d7d8e9a090c748800d69a57bbd9af698b60 + PEGLEG_IMAGE_DISTRO: ubuntu_focal + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ + - ^releasenotes/.*$ + - job: name: airship-pegleg-docker-build-gate-ubuntu_focal - timeout: 3600 + timeout: 7200 run: tools/gate/playbooks/docker-image-build.yaml nodeset: airship-pegleg-single-node-focal irrelevant-files: @@ -105,7 +144,7 @@ - job: name: airship-pegleg-docker-publish-ubuntu_focal - timeout: 3600 + timeout: 7200 run: tools/gate/playbooks/docker-image-build.yaml nodeset: airship-pegleg-single-node-focal secrets: @@ -121,39 +160,7 @@ commit: true static: - latest - -- job: - name: airship-pegleg-docker-build-gate-ubuntu_bionic - timeout: 3600 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: airship-pegleg-single-node-focal - irrelevant-files: - - '^doc/.*' - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true - -- job: - name: airship-pegleg-docker-publish-ubuntu_bionic - timeout: 3600 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: airship-pegleg-single-node-focal - secrets: - - airship_pegleg_quay_creds - irrelevant-files: - - '^doc/.*' - vars: - publish: true - distro: ubuntu_bionic - tags: - dynamic: - branch: true - commit: true - static: - - latest + - airflow_2.6.2 - secret: name: airship_pegleg_quay_creds diff --git a/doc/source/images/architecture-pegleg.png b/doc/source/images/architecture-pegleg.png index 719d2899..18ae9fd2 100644 Binary files a/doc/source/images/architecture-pegleg.png and b/doc/source/images/architecture-pegleg.png differ diff --git a/requirements-direct.txt b/requirements-direct.txt index d4928c67..4af41f39 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -1,16 +1,14 @@ -click==7.1.2 jsonschema -cryptography==3.4.8 -python-dateutil==2.8.1 +python-dateutil GitPython docker -pylibyaml==0.1.0 -PyYAML<=5.4.1 -deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand -shipyard-client @ git+https://opendev.org/airship/shipyard.git@7ea83c25efc4e689e5cb0df19614d9ca538eda22#egg=shipyard-client&subdirectory=src/bin/shipyard_client -promenade @ git+https://opendev.org/airship/promenade.git@16debd81747fcde8e0628e141ee67d697b06c46d#egg=promenade -oslo.i18n -oslo.utils<=4.12.3 +pylibyaml +PyYAML +deckhand @ git+https://opendev.org/airship/deckhand.git@37ba567da5bf7f146f54a468478a025c70793574#egg=deckhand +shipyard-client @ git+https://opendev.org/airship/shipyard.git@81066ae98fe2afd3a9c8c5c8556e9438ac47d5a2#egg=shipyard-client&subdirectory=src/bin/shipyard_client +promenade @ git+https://opendev.org/airship/promenade.git@69a74590e76e810916f7780fc525c63ec58c7dc1#egg=promenade +oslo.i18n==6.0.0 +oslo.utils==4.12.3 six debtcollector zipp diff --git a/requirements-frozen-bionic.txt b/requirements-frozen-bionic.txt deleted file mode 100644 index 8fef4179..00000000 --- a/requirements-frozen-bionic.txt +++ /dev/null @@ -1,177 +0,0 @@ -alabaster==0.7.13 -alembic==1.4.3 -amqp==2.6.1 -argcomplete==3.0.8 -arrow==0.17.0 -attrs==22.2.0 -autopage==0.5.1 -Babel==2.11.0 -bcrypt==4.0.1 -Beaker==1.12.0 -cachetools==4.2.4 -certifi==2023.5.7 -cffi==1.15.1 -chardet==3.0.4 -charset-normalizer==3.0.1 -click==7.1.2 -click-default-group==1.2 -cliff==3.10.1 -cmd2==2.4.3 -configparser==5.2.0 -coverage==5.3 -cryptography==3.4.8 -debtcollector==2.5.0 -Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5 -decorator==4.4.2 -deepdiff==5.7.0 -distlib==0.3.6 -dnspython==2.2.1 -docker==5.0.3 -docutils==0.17 -dogpile.cache==1.2.1 -dulwich==0.20.50 -eventlet==0.33.3 -extras==1.0.0 -falcon==3.1.1 -fasteners==0.18 -filelock==3.4.1 -fixtures==3.0.0 -flake8==3.8.4 -Flask==1.1.0 -futurist==2.4.1 -gitdb==4.0.9 -GitPython==3.1.18 -google-auth==2.19.1 -greenlet==2.0.2 -hacking==4.1.0 -html5lib==0.9999999 -httpexceptor==1.4.0 -idna==2.10 -imagesize==1.4.1 -importlib-metadata==2.1.3 -importlib-resources==5.4.0 -iso8601==1.1.0 -itsdangerous==2.0.1 -Jinja2==2.10.3 -jsonpath-ng==1.5.3 -jsonpath-rw==1.4.0 -jsonpath-rw-ext==1.2.2 -jsonpickle==1.4.1 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==9.5.0 -kombu==4.6.11 -kubernetes==26.1.0 -Mako==1.1.6 -MarkupSafe==2.0.1 -mccabe==0.6.1 -mock==5.0.2 -msgpack==1.0.5 -netaddr==0.8.0 -netifaces==0.11.0 -networkx==2.5.1 -nose==1.3.7 -oauthlib==3.2.2 -ordered-set==4.0.2 -os-service-types==1.7.0 -oslo.cache==2.10.1 -oslo.concurrency==4.5.1 -oslo.config==8.7.1 -oslo.context==4.1.0 -oslo.db==10.0.0 -oslo.i18n==5.1.0 -oslo.log==4.6.0 -oslo.messaging==12.13.0 -oslo.metrics==0.4.0 -oslo.middleware==4.4.0 -oslo.policy==3.10.1 -oslo.serialization==4.2.0 -oslo.service==2.8.0 -oslo.utils==4.12.3 -packaging==21.3 -Paste==3.5.0 -PasteDeploy==2.1.1 -PasteScript==3.3.0 -pbr==5.5.1 -pip==21.3.1 -platformdirs==2.4.0 -ply==3.11 -prettytable==2.5.0 -promenade @ git+https://opendev.org/airship/promenade.git@16debd81747fcde8e0628e141ee67d697b06c46d -prometheus-client==0.17.0 -psycopg2-binary==2.9.6 -PTable==0.9.2 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 -pycadf==3.1.1 -pycodestyle==2.6.0 -pycparser==2.21 -pyflakes==2.2.0 -Pygments==2.14.0 -pylibyaml==0.1.0 -pymongo==4.1.1 -pyparsing==2.4.7 -pyperclip==1.8.2 -pyproject-api==0.1.0 -pyrsistent==0.18.0 -python-barbicanclient==5.2.0 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-keystoneclient==4.5.0 -python-memcached==1.59 -python-mimeparse==1.6.0 -python-subunit==1.4.0 -pytz==2023.3 -PyYAML==5.4.1 -reno==4.0.0 -repoze.lru==0.7 -requests==2.23.0 -requests-oauthlib==1.3.1 -resolver==0.2.1 -rfc3986==1.5.0 -Routes==2.5.1 -rsa==4.9 -selector==0.10.1 -setuptools==45.2.0 -shipyard-client @ git+https://opendev.org/airship/shipyard.git@7ea83c25efc4e689e5cb0df19614d9ca538eda22#subdirectory=src/bin/shipyard_client -simplejson==3.19.1 -six==1.16.0 -smmap==5.0.0 -snowballstemmer==2.2.0 -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==1.3.20 -sqlalchemy-migrate==0.13.0 -sqlparse==0.4.4 -statsd==4.0.1 -stevedore==3.5.2 -Tempita==0.5.2 -testrepository==0.0.20 -testresources==2.0.1 -testscenarios==0.5.0 -testtools==2.5.0 -tiddlyweb==2.4.3 -tomli==1.2.3 -tomlkit==0.11.6 -typing-extensions==3.7.4.3 -urllib3==1.25.11 -uWSGI==2.0.21 -vine==1.3.0 -virtualenv==20.16.2 -wcwidth==0.2.6 -WebOb==1.8.7 -websocket-client==1.3.1 -Werkzeug==0.16.1 -wheel==0.37.1 -wrapt==1.15.0 -wsgi-intercept==1.11.0 -xmltodict==0.13.0 -yappi==1.4.0 -yq==3.2.1 -zipp==3.6.0 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 21013de9..16727fda 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,176 +1,144 @@ -alabaster==0.7.13 -alembic==1.4.3 -amqp==2.6.1 -argcomplete==3.0.8 -arrow==0.17.0 +alembic==1.11.1 +amqp==5.1.1 +arrow==1.2.3 attrs==23.1.0 autopage==0.5.1 -Babel==2.12.1 +backports.zoneinfo==0.2.1 +barbican==14.0.2 bcrypt==4.0.1 -Beaker==1.12.0 +Beaker==1.12.1 cachetools==5.3.1 +castellan==4.1.0 certifi==2023.5.7 cffi==1.15.1 -chardet==3.0.4 charset-normalizer==3.1.0 -click==7.1.2 -click-default-group==1.2 -cliff==3.10.1 +click==8.1.3 +click-default-group==1.2.2 +cliff==4.3.0 cmd2==2.4.3 -configparser==5.3.0 -coverage==5.3 -cryptography==3.4.8 +cryptography==40.0.2 debtcollector==2.5.0 -Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5 +Deckhand @ git+https://opendev.org/airship/deckhand.git@37ba567da5bf7f146f54a468478a025c70793574 decorator==5.1.1 -deepdiff==5.8.1 -distlib==0.3.6 +deepdiff==6.3.1 dnspython==2.3.0 -docker==5.0.3 -docutils==0.17 -dogpile.cache==1.2.1 -dulwich==0.21.5 +docker==6.1.3 +dogpile.cache==1.2.2 eventlet==0.33.3 -extras==1.0.0 falcon==3.1.1 fasteners==0.18 -filelock==3.12.0 -fixtures==3.0.0 -flake8==3.8.4 -Flask==1.1.0 +fixtures==4.1.0 +Flask==2.2.5 futurist==2.4.1 gitdb==4.0.10 GitPython==3.1.31 -google-auth==2.19.1 +google-auth==2.20.0 greenlet==2.0.2 -hacking==4.1.0 html5lib==0.9999999 httpexceptor==1.4.0 -idna==2.10 -imagesize==1.4.1 -importlib-metadata==2.1.3 +idna==3.4 +importlib-metadata==4.13.0 +importlib-resources==5.12.0 iso8601==2.0.0 -itsdangerous==2.0.1 -Jinja2==2.10.3 +itsdangerous==2.1.2 +Jinja2==3.1.2 jsonpath-ng==1.5.3 -jsonpath-rw==1.4.0 -jsonpath-rw-ext==1.2.2 -jsonpickle==1.4.1 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==10.3.0 -kombu==4.6.11 -kubernetes==26.1.0 +jsonpickle==3.0.1 +jsonschema==4.17.3 +keystoneauth1==4.3.1 +keystonemiddleware==10.4.0 +kombu==5.3.0 +kubernetes==23.6.0 +ldap3==2.9.1 +logutils==0.3.5 Mako==1.2.4 -MarkupSafe==2.0.1 -mccabe==0.6.1 -mock==5.0.2 +MarkupSafe==2.1.3 msgpack==1.0.5 netaddr==0.8.0 netifaces==0.11.0 networkx==3.1 -nose==1.3.7 oauthlib==3.2.2 ordered-set==4.1.0 os-service-types==1.7.0 -oslo.cache==2.10.1 +oslo.cache==3.4.0 oslo.concurrency==5.1.1 -oslo.config==8.7.1 -oslo.context==4.1.0 -oslo.db==10.0.0 +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.db==11.2.0 oslo.i18n==6.0.0 -oslo.log==4.6.0 -oslo.messaging==12.13.0 +oslo.log==4.8.0 +oslo.messaging==14.3.1 oslo.metrics==0.6.0 -oslo.middleware==4.4.0 +oslo.middleware==5.1.1 oslo.policy==3.10.1 -oslo.serialization==4.2.0 +oslo.serialization==4.1.0 oslo.service==3.1.1 +oslo.upgradecheck==2.1.1 oslo.utils==4.12.3 +oslo.versionedobjects==3.1.0 packaging==21.3 -Paste==3.5.0 +Paste==3.5.3 PasteDeploy==3.0.1 PasteScript==3.3.0 -pbr==5.5.1 +pbr==5.11.1 +pecan==1.5.1 pip==23.1.2 -platformdirs==3.5.1 +pkgutil_resolve_name==1.3.10 ply==3.11 -prettytable==3.7.0 -promenade @ git+https://opendev.org/airship/promenade.git@16debd81747fcde8e0628e141ee67d697b06c46d +prettytable==3.8.0 +promenade @ git+https://opendev.org/airship/promenade.git@69a74590e76e810916f7780fc525c63ec58c7dc1 prometheus-client==0.17.0 psycopg2-binary==2.9.6 PTable==0.9.2 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 pycadf==3.1.1 -pycodestyle==2.6.0 pycparser==2.21 -pyflakes==2.2.0 -Pygments==2.14.0 pylibyaml==0.1.0 -pymongo==4.3.3 -pyparsing==2.4.7 +pyOpenSSL==23.2.0 +pyparsing==3.0.9 pyperclip==1.8.2 -pyproject_api==1.5.0 pyrsistent==0.19.3 -python-barbicanclient==5.2.0 -python-dateutil==2.8.1 -python-editor==1.0.4 +python-barbicanclient==5.4.0 +python-dateutil==2.8.2 python-keystoneclient==5.1.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-subunit==1.4.0 pytz==2023.3 -PyYAML==5.4.1 -reno==4.0.0 +PyYAML==6.0 repoze.lru==0.7 -requests==2.23.0 +requests==2.31.0 requests-oauthlib==1.3.1 resolver==0.2.1 -rfc3986==2.0.0 +rfc3986==1.5.0 Routes==2.5.1 rsa==4.9 selector==0.10.1 -setuptools==45.2.0 -shipyard-client @ git+https://opendev.org/airship/shipyard.git@7ea83c25efc4e689e5cb0df19614d9ca538eda22#subdirectory=src/bin/shipyard_client +setuptools==67.7.2 +shipyard-client @ git+https://opendev.org/airship/shipyard.git@81066ae98fe2afd3a9c8c5c8556e9438ac47d5a2#subdirectory=src/bin/shipyard_client simplejson==3.19.1 six==1.16.0 smmap==5.0.0 -snowballstemmer==2.2.0 -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==1.3.20 +SQLAlchemy==1.4.48 sqlalchemy-migrate==0.13.0 sqlparse==0.4.4 statsd==4.0.1 stevedore==5.1.0 Tempita==0.5.2 -testrepository==0.0.20 testresources==2.0.1 testscenarios==0.5.0 -testtools==2.5.0 +testtools==2.6.0 tiddlyweb==2.4.3 -tomli==2.0.1 -tomlkit==0.11.8 -typing-extensions==3.7.4.3 -urllib3==1.25.11 -uWSGI==2.0.21 -vine==1.3.0 -virtualenv==20.23.0 +typing_extensions==4.6.3 +tzdata==2023.3 +urllib3==1.26.16 +uWSGI==2.0.22 +vine==5.0.0 wcwidth==0.2.6 WebOb==1.8.7 websocket-client==1.5.3 -Werkzeug==0.16.1 +Werkzeug==2.2.3 wheel==0.40.0 wrapt==1.15.0 -wsgi-intercept==1.11.0 -xmltodict==0.13.0 yappi==1.4.0 -yq==3.2.1 zipp==3.15.0 diff --git a/tools/gate/playbooks/airskiff-reduce-site.yaml b/tools/gate/playbooks/airskiff-reduce-site.yaml new file mode 100644 index 00000000..7c80ed77 --- /dev/null +++ b/tools/gate/playbooks/airskiff-reduce-site.yaml @@ -0,0 +1,23 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Overwrite Armada manifest + shell: | + git checkout v1.9 + mv tools/gate/manifests/full-site.yaml \ + type/skiff/manifests/full-site.yaml + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" diff --git a/tools/gate/playbooks/airskiff-validate.yaml b/tools/gate/playbooks/airskiff-validate.yaml new file mode 100644 index 00000000..799bc63a --- /dev/null +++ b/tools/gate/playbooks/airskiff-validate.yaml @@ -0,0 +1,152 @@ +# Copyright 2019, AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + roles: + - clear-firewall + - bindep + - disable-systemd-resolved + - ensure-python + - ensure-pip + - ensure-docker + + tasks: + + - name: Install Packaging python module for tools/airship + block: + - pip: + name: packaging + version: 23.1 + executable: pip3 + become: True + + - name: Clone Required Repositories + shell: | + export CLONE_PEGLEG={{ CLONE_PEGLEG }} + export OSH_INFRA_COMMIT={{ HTK_COMMIT }} + ./tools/deployment/airskiff/developer/000-clone-dependencies.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + + - name: Setup Apparmor + shell: | + ./tools/deployment/airskiff/developer/009-setup-apparmor.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + + - name: Deploy Kubernetes with Minikube + shell: | + ./tools/deployment/airskiff/developer/010-deploy-k8s.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + + + - name: Build all charts locally + shell: | + set -ex + ./tools/deployment/airskiff/developer/015-make-all-charts.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: Build Pegleg with submitted changes + shell: | + set -ex + export DISTRO={{ PEGLEG_IMAGE_DISTRO }} + make images + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes + + + - name: Use locally built images in manifests + shell: | + set -ex + docker rm registry --force || true + docker run -d -p 5000:5000 --restart=always --name registry registry:2 + docker tag quay.io/airshipit/pegleg:latest-ubuntu_focal localhost:5000/pegleg:latest-ubuntu_focal + docker push localhost:5000/pegleg:latest-ubuntu_focal + sed -i "s#quay.io/airshipit/pegleg:latest-ubuntu_focal#localhost:5000/pegleg:latest-ubuntu_focal#g" ./global/software/config/versions.yaml + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: Start artifactory + shell: | + set -ex + # start http server with artifacts + docker rm artifacts --force || true + docker run --name artifacts -p 8282:80 -v $(pwd)/../artifacts:/usr/share/nginx/html -d nginx + sleep 10 + curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: List sites + shell: | + set -xe + ./tools/airship pegleg --verbose site -r . list + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: Lint Airskiff site + shell: | + set -xe + ./tools/airship pegleg --verbose site -r . lint airskiff -w P001 -w P003 + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: Validate Airskiff site + shell: | + set -xe + ./tools/airship pegleg --verbose site -r . render airskiff --validate + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: Render Airskiff site + shell: | + set -xe + ./tools/airship pegleg --verbose site -r . render airskiff -o airskiff.yaml + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: Show Airskiff site + shell: | + set -xe + ./tools/airship pegleg --verbose site -r . show airskiff + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: Collect Airskiff site + shell: | + set -xe + ./tools/airship pegleg --verbose site -r . collect airskiff -s peggles + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + + - name: Stop artifactory + shell: | + set -ex + # terminate artifactory + docker rm artifacts --force || true + args: + chdir: "{{ zuul.project.src_dir }}" + become: yes diff --git a/tools/gate/playbooks/debug-report.yaml b/tools/gate/playbooks/debug-report.yaml new file mode 100644 index 00000000..8404a3a6 --- /dev/null +++ b/tools/gate/playbooks/debug-report.yaml @@ -0,0 +1,34 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Build debug report + shell: | + ./tools/gate/debug-report.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + + - name: Extract debug report + shell: | + mkdir -p /tmp/debug + tar -xf debug-* -C /tmp/debug + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + + - name: Pull logs to executor + synchronize: + src: /tmp/debug + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull diff --git a/tools/gate/playbooks/git-config.yaml b/tools/gate/playbooks/git-config.yaml new file mode 100644 index 00000000..d4a118e5 --- /dev/null +++ b/tools/gate/playbooks/git-config.yaml @@ -0,0 +1,25 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Git config + shell: | + set -xe; + tee .git/config << EOF + [remote "origin"] + url = https://opendev.org/airship/treasuremap.git + EOF + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" diff --git a/tools/gate/playbooks/roles b/tools/gate/playbooks/roles new file mode 120000 index 00000000..d8c4472c --- /dev/null +++ b/tools/gate/playbooks/roles @@ -0,0 +1 @@ +../roles \ No newline at end of file diff --git a/tools/gate/roles/deploy-package/defaults/main.yml b/tools/gate/roles/deploy-package/defaults/main.yml new file mode 100644 index 00000000..b1a6fabd --- /dev/null +++ b/tools/gate/roles/deploy-package/defaults/main.yml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +proxy: + http: null + https: null + noproxy: null +... diff --git a/tools/gate/roles/deploy-package/tasks/dist.yaml b/tools/gate/roles/deploy-package/tasks/dist.yaml new file mode 100644 index 00000000..73939ffd --- /dev/null +++ b/tools/gate/roles/deploy-package/tasks/dist.yaml @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: managing distro packages for ubuntu + become: true + become_user: root + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + vars: + state: present + apt: + name: "{{ item }}" + state: "{{ state }}" + with_items: "{{ packages.deb }}" + +- name: managing distro packages for centos + become: true + become_user: root + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + vars: + state: present + yum: + name: "{{ item }}" + state: "{{ state }}" + with_items: "{{ packages.rpm }}" + +- name: managing distro packages for fedora + become: true + become_user: root + when: ansible_distribution == 'Fedora' + vars: + state: present + dnf: + name: "{{ item }}" + state: "{{ state }}" + with_items: "{{ packages.rpm }}" +... diff --git a/tools/gate/roles/deploy-package/tasks/pip.yaml b/tools/gate/roles/deploy-package/tasks/pip.yaml new file mode 100644 index 00000000..0b2a4836 --- /dev/null +++ b/tools/gate/roles/deploy-package/tasks/pip.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: managing pip packages + become: true + become_user: root + environment: + http_proxy: "{{ proxy.http }}" + https_proxy: "{{ proxy.https }}" + no_proxy: "{{ proxy.noproxy }}" + vars: + state: present + pip: + name: "{{ item }}" + state: "{{ state }}" + with_items: "{{ packages }}" +... diff --git a/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml b/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml new file mode 100644 index 00000000..0eda1fbf --- /dev/null +++ b/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml @@ -0,0 +1,37 @@ +# Copyright 2020 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Disable systemd-resolved service + systemd: + state: stopped + enabled: no + masked: yes + daemon_reload: yes + name: systemd-resolved + become: yes + +- name: Remove local stub dns from resolv.conf, if it exists + lineinfile: + path: /etc/resolv.conf + state: absent + regexp: '^nameserver.*127.0.0.1' + become: yes + +- name: Add upstream nameservers in resolv.conf + blockinfile: + path: /etc/resolv.conf + block: | + nameserver 8.8.8.8 + nameserver 8.8.4.4 + become: yes diff --git a/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml b/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml new file mode 100644 index 00000000..bb381b41 --- /dev/null +++ b/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml @@ -0,0 +1,15 @@ +# Copyright 2020 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: disable-systemd-resolved.yaml diff --git a/tox.ini b/tox.ini index a7f25a99..c85f4900 100644 --- a/tox.ini +++ b/tox.ini @@ -30,16 +30,6 @@ commands = bash -c "{toxinidir}/tools/install-cfssl.sh" bash -c "{toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'" -[testenv:py36] -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -allowlist_externals = - bash - find -commands = - {[testenv]commands} - [testenv:py38] allowlist_externals = @@ -103,17 +93,6 @@ allowlist_externals = bash find -[testenv:freeze-bionic] -recreate = True -allowlist_externals= - rm - sh -deps= - -r{toxinidir}/requirements-direct.txt -commands= - rm -f {toxinidir}/requirements-frozen-bionic.txt - sh -c "pip freeze --all | grep -vE 'pegleg|pyinotify|pkg-resources' > requirements-frozen-bionic.txt" - [testenv:freeze] recreate = True allowlist_externals= @@ -121,6 +100,7 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt commands= rm -f {toxinidir}/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'pegleg|pyinotify|pkg-resources' > requirements-frozen.txt" @@ -130,7 +110,6 @@ commands= [testenv:releasenotes] basepython = python3 deps= - -r{toxinidir}/requirements-frozen.txt -r{toxinidir}/test-requirements.txt -r{toxinidir}/doc/requirements.txt commands = diff --git a/vcs-requirements.env b/vcs-requirements.env index 46bc200d..13b1ecfe 100644 --- a/vcs-requirements.env +++ b/vcs-requirements.env @@ -1,3 +1,3 @@ -DECKHAND_VERSION=commit/06b1631d8a31a1e7147e8751025d0ef2ff4b31f5 -SHIPYARD_VERSION=commit/7ea83c25efc4e689e5cb0df19614d9ca538eda22 -PROMENADE_VERSION=commit/16debd81747fcde8e0628e141ee67d697b06c46d +DECKHAND_VERSION=commit/37ba567da5bf7f146f54a468478a025c70793574 +SHIPYARD_VERSION=commit/81066ae98fe2afd3a9c8c5c8556e9438ac47d5a2 +PROMENADE_VERSION=commit/69a74590e76e810916f7780fc525c63ec58c7dc1