From db5aa78b9e5622af99bab26b972f09ba512ccc9a Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Sat, 1 Jul 2023 02:08:39 +0000 Subject: [PATCH] Airflow stable 2.6.2 This PS updates python modules and code to match Airflow 2.6.2 Change-Id: I65a1b86473ee3e988aae353b59fb5473d75851f9 --- .zuul.yaml | 216 +++--------------- ChangeLog | 3 + charts/deckhand/values.yaml | 2 +- deckhand/conf/config.py | 6 +- deckhand/tests/common/test_gabbi.py | 2 +- entrypoint.sh | 6 +- etc/deckhand/deckhand.conf.sample | 11 +- print_sys_path.py | 4 + requirements-direct.txt | 90 ++------ requirements-frozen-bionic.txt | 156 ------------- requirements-frozen.txt | 151 +++++------- test-requirements.txt | 30 +-- tools/gate/playbooks/airskiff-deploy.yaml | 10 +- .../gate/playbooks/airskiff-reduce-site.yaml | 1 + tools/gate/playbooks/git-config.yaml | 2 +- .../tasks/install-test-requirements.yaml | 22 +- .../tasks/integration-tests.yaml | 11 +- tools/gate/scripts/020-deploy-postgresql.sh | 2 +- tools/integration-tests.sh | 84 +++++-- tox.ini | 81 +------ 20 files changed, 232 insertions(+), 658 deletions(-) create mode 100644 print_sys_path.py delete mode 100644 requirements-frozen-bionic.txt diff --git a/.zuul.yaml b/.zuul.yaml index 468988e4..68183237 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -12,57 +12,43 @@ - project: templates: - - openstack-cover-jobs - docs-on-readthedocs - openstack-python38-jobs - - openstack-python36-jobs vars: rtd_webhook_id: '38572' rtd_project_name: 'airship-deckhand' check: jobs: - - deckhand-tox-py36-postgresql - deckhand-tox-py38-postgresql - - deckhand-functional-uwsgi-py36 - deckhand-functional-uwsgi-py38 - - deckhand-functional-docker-py38-ubuntu_focal - - deckhand-functional-docker-py36-ubuntu_bionic - - deckhand-integration-uwsgi-py36 + - deckhand-functional-docker-py38 - deckhand-integration-uwsgi-py38 - - deckhand-integration-docker-py38-ubuntu_focal - - deckhand-integration-docker-py36-ubuntu_bionic + - deckhand-integration-docker-py38 - deckhand-chart-build-gate - deckhand-chart-build-latest-htk - - deckhand-docker-build-gate-ubuntu_focal - - deckhand-docker-build-gate-ubuntu_bionic - - openstack-tox-pep8 + - deckhand-docker-build-gate + - deckhand-openstack-tox-pep8-focal + - deckhand-openstack-tox-cover-focal - deckhand-airskiff-deployment-focal - - deckhand-airskiff-deployment-bionic + gate: jobs: - - deckhand-tox-py36-postgresql - deckhand-tox-py38-postgresql - - deckhand-functional-uwsgi-py36 - deckhand-functional-uwsgi-py38 - - deckhand-functional-docker-py38-ubuntu_focal - - deckhand-functional-docker-py36-ubuntu_bionic - - deckhand-integration-uwsgi-py36 + - deckhand-functional-docker-py38 - deckhand-integration-uwsgi-py38 - - deckhand-integration-docker-py38-ubuntu_focal - - deckhand-integration-docker-py36-ubuntu_bionic + - deckhand-integration-docker-py38 - deckhand-chart-build-gate - - deckhand-docker-build-gate-ubuntu_focal - - deckhand-docker-build-gate-ubuntu_bionic - - openstack-tox-pep8 + - deckhand-docker-build-gate + - deckhand-openstack-tox-pep8-focal + - deckhand-openstack-tox-cover-focal - deckhand-airskiff-deployment-focal - - deckhand-airskiff-deployment-bionic + post: jobs: - deckhand-upload-git-mirror - deckhand-docker-publish-ubuntu_focal - deckhand-docker-tag-ubuntu_focal - - deckhand-docker-publish-ubuntu_bionic - - deckhand-docker-tag-ubuntu_bionic - nodeset: @@ -89,6 +75,18 @@ - name: primary label: ubuntu-focal +- job: + name: deckhand-openstack-tox-pep8-focal + parent: openstack-tox-pep8 + description: Runs pep8 job on focal + nodeset: deckhand-single-node-airskiff-focal + +- job: + name: deckhand-openstack-tox-cover-focal + parent: openstack-tox-cover + description: Runs cover job on focal + nodeset: deckhand-single-node-airskiff-focal + - job: name: deckhand-tox-py38-postgresql parent: openstack-tox-py38 @@ -98,35 +96,6 @@ vars: tox_envlist: py38-postgresql -- job: - name: deckhand-tox-py36-postgresql - parent: openstack-tox-py38 - nodeset: deckhand-single-node - pre-run: - - tools/gate/playbooks/install-postgresql.yaml - vars: - tox_envlist: py36-postgresql - -- job: - name: deckhand-functional-uwsgi-py36 - voting: true - description: | - Run tox-based functional tests for the Airship Deckhand project using a - minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf - for ephemeral PostgreSQL DB, under cPython version 3.6. - run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml - nodeset: deckhand-single-node - vars: - tox_envlist: py36-functional-dev - distro: ubuntu_bionic - irrelevant-files: &irrelevant-files - - ^.*\.rst$ - - ^doc/.*$ - - ^etc/.*$ - - ^releasenotes/.*$ - - ^setup.cfg$ - - ^deckhand/tests/unit/.*$ - - job: name: deckhand-functional-uwsgi-py38 @@ -140,7 +109,13 @@ vars: tox_envlist: py38-functional-dev distro: ubuntu_focal - irrelevant-files: *irrelevant-files + irrelevant-files: &irrelevant-files + - ^.*\.rst$ + - ^doc/.*$ + - ^etc/.*$ + - ^releasenotes/.*$ + - ^setup.cfg$ + - ^deckhand/tests/unit/.*$ - job: name: deckhand-functional-docker-base @@ -159,7 +134,7 @@ - openstack/openstack-helm-infra - job: - name: deckhand-functional-docker-py38-ubuntu_focal + name: deckhand-functional-docker-py38 description: | Run tox-based functional tests for the Airship Deckhand project under cPython version 3.8. Uses tox with the ``functional-py38`` environment. @@ -172,36 +147,6 @@ distro: ubuntu_focal irrelevant-files: *irrelevant-files -- job: - name: deckhand-functional-docker-py36-ubuntu_bionic - description: | - Run tox-based functional tests for the Airship Deckhand project under - cPython version 3.6. Uses tox with the ``functional-py36`` environment. - Ubuntu (focal) image is built and used. - parent: deckhand-functional-docker-base - nodeset: deckhand-single-node - vars: - tox_envlist: py36-functional - disable_keystone: true - distro: ubuntu_bionic - irrelevant-files: *irrelevant-files - - - -- job: - name: deckhand-integration-uwsgi-py36 - voting: true - description: | - Run tox-based integration tests for the Airship Deckhand project using a - minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf - for ephemeral PostgreSQL DB, under cPython version 3.6. - timeout: 3600 - run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml - nodeset: deckhand-single-node - irrelevant-files: *irrelevant-files - vars: - disable_keystone: true - distro: ubuntu_bionic - job: name: deckhand-integration-uwsgi-py38 @@ -210,7 +155,7 @@ Run tox-based integration tests for the Airship Deckhand project using a minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf for ephemeral PostgreSQL DB, under cPython version 3.8. - timeout: 3600 + timeout: 9600 run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml nodeset: deckhand-single-node-focal irrelevant-files: *irrelevant-files @@ -263,7 +208,7 @@ - ^releasenotes/.*$ - job: - name: deckhand-integration-docker-py38-ubuntu_focal + name: deckhand-integration-docker-py38 description: | Run tox-based integration tests for the Airship Deckhand project under cPython version 3.8. Builds ubuntu (focal) deckhand image. @@ -273,17 +218,6 @@ disable_keystone: false distro: ubuntu_focal -- job: - name: deckhand-integration-docker-py36-ubuntu_bionic - description: | - Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.6. Builds ubuntu (focal) deckhand image. - parent: deckhand-integration-docker-base - nodeset: openstack-helm-single-node - vars: - disable_keystone: false - distro: ubuntu_focal - - job: name: deckhand-airskiff-deployment-focal voting: true @@ -292,15 +226,15 @@ Deploy Memcached using Airskiff and submitted Deckhand changes. timeout: 9600 pre-run: - - tools/gate/playbooks/airskiff-reduce-site.yaml - tools/gate/playbooks/git-config.yaml + - tools/gate/playbooks/airskiff-reduce-site.yaml run: tools/gate/playbooks/airskiff-deploy.yaml post-run: tools/gate/playbooks/debug-report.yaml required-projects: - airship/treasuremap vars: CLONE_DECKHAND: false - OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74 + OSH_INFRA_COMMIT: 3d8935a536faaa87e084e7783ed53f4660a3a1f8 DECKHAND_IMAGE_DISTRO: ubuntu_focal distro: ubuntu_focal irrelevant-files: @@ -308,33 +242,9 @@ - ^doc/.*$ - ^releasenotes/.*$ -- job: - name: deckhand-airskiff-deployment-bionic - voting: true - nodeset: deckhand-single-node-airskiff-focal - description: | - Deploy Memcached using Airskiff and submitted Deckhand changes. - timeout: 9600 - pre-run: - - tools/gate/playbooks/airskiff-reduce-site.yaml - - tools/gate/playbooks/git-config.yaml - run: tools/gate/playbooks/airskiff-deploy.yaml - post-run: tools/gate/playbooks/debug-report.yaml - required-projects: - - airship/treasuremap - vars: - CLONE_DECKHAND: false - OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74 - DECKHAND_IMAGE_DISTRO: ubuntu_bionic - distro: ubuntu_focal - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^releasenotes/.*$ - - job: - name: deckhand-docker-build-gate-ubuntu_focal + name: deckhand-docker-build-gate timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml nodeset: deckhand-single-node-focal @@ -352,19 +262,6 @@ dynamic: patch_set: true -- job: - name: deckhand-docker-build-gate-ubuntu_bionic - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - irrelevant-files: *non-code-files-template - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true - - job: name: deckhand-docker-publish-ubuntu_focal description: | @@ -387,28 +284,6 @@ static: - latest -- job: - name: deckhand-docker-publish-ubuntu_bionic - description: | - Runs on every merge, unless files in a dictionary below are changed. - Builds and publishes container ubuntu images on quay.io with a set of tags - listed in vars section. Waits in Zuul queue for a node (VM) assignment. - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - secrets: - - airship_deckhand_quay_creds - irrelevant-files: *non-code-files-template - vars: - publish: true - distro: ubuntu_bionic - tags: - dynamic: - branch: true - commit: true - static: - - latest - - job: name: deckhand-docker-tag-ubuntu_focal description: | @@ -425,23 +300,6 @@ vars: distro: ubuntu_focal -- job: - name: deckhand-docker-tag-ubuntu_bionic - description: | - Runs on every merge when files in a dictionalry below are changed, and - adds git commit id tag onto the ubuntu container image published on quay.io, - which has `latest` tag set. Does not wait in queue for a node (VM) - assignment, runs almost immediately. - timeout: 1800 - run: tools/gate/playbooks/docker-image-tag.yaml - nodeset: - nodes: [] - secrets: - - airship_deckhand_quay_creds - vars: - distro: ubuntu_bionic - - - secret: name: airship_deckhand_quay_creds diff --git a/ChangeLog b/ChangeLog index ef7422a8..7cedf51a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,9 @@ CHANGES ======= +* Airflow stable 2.6.2 +* Restored ubuntu\_bionic image build +* [focal] Fix requests.body attribute deprecation * Update airskiff deployment gate * Deckhand updates * Removing egg-info folder diff --git a/charts/deckhand/values.yaml b/charts/deckhand/values.yaml index 88a406be..8f6115ae 100644 --- a/charts/deckhand/values.yaml +++ b/charts/deckhand/values.yaml @@ -29,7 +29,7 @@ images: tags: deckhand: quay.io/airshipit/deckhand:latestlatest-ubuntu_focal dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 - db_init: docker.io/postgres:14.6 + db_init: docker.io/postgres:14.8 db_sync: quay.io/airshipit/deckhand:latest-ubuntu_focal image_repo_sync: docker.io/docker:23.0.3 ks_endpoints: docker.io/openstackhelm/heat:wallaby-ubuntu_focal diff --git a/deckhand/conf/config.py b/deckhand/conf/config.py index f405d38f..9d9de70c 100644 --- a/deckhand/conf/config.py +++ b/deckhand/conf/config.py @@ -43,7 +43,7 @@ barbican_opts = [ help="Whether to enable Barbican secret caching. Useful " "for testing to avoid cross-test caching conflicts."), cfg.StrOpt( - 'cache_timeout', default=3600, + 'cache_timeout', default='3600', help="How long (in seconds) Barbican secret reference/payload lookup " "results should remain cached in memory.") ] @@ -63,7 +63,7 @@ engine_opts = [ cfg.BoolOpt('enable_cache', default=True, help="Whether to enable the document rendering caching. Useful" " for testing to avoid cross-test caching conflicts."), - cfg.IntOpt('cache_timeout', default=3600, + cfg.IntOpt('cache_timeout', default='3600', help="How long (in seconds) document rendering results should " "remain cached in memory."), ] @@ -76,7 +76,7 @@ jsonpath_group = cfg.OptGroup( jsonpath_opts = [ - cfg.IntOpt('cache_timeout', default=3600, + cfg.IntOpt('cache_timeout', default='3600', help="How long (in seconds) JSONPath lookup results should " "remain cached in memory.") ] diff --git a/deckhand/tests/common/test_gabbi.py b/deckhand/tests/common/test_gabbi.py index 647fc780..6624671d 100644 --- a/deckhand/tests/common/test_gabbi.py +++ b/deckhand/tests/common/test_gabbi.py @@ -90,7 +90,7 @@ class MultidocJsonpaths(jsonhandler.JSONHandler): # NOTE: The simple approach to handling dictionary versus list response # bodies is to always parse the response body as a list and index into # the first element using [0] throughout the tests. - return list(yaml.load_all(string)) + return list(yaml.safe_load_all(string)) def load_tests(loader, tests, pattern): diff --git a/entrypoint.sh b/entrypoint.sh index 2f43b2d4..0751be80 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -38,13 +38,13 @@ echo "Command: $1 with arguments $@" # Start deckhand application if [ "$1" = 'server' ]; then exec uwsgi \ - -b 32768 \ + --buffer-size 32768 \ --callable deckhand_callable \ --die-on-term \ --enable-threads \ --http :${PORT} \ --http-timeout ${DECKHAND_API_TIMEOUT} \ - -L \ + --disable-logging \ --lazy-apps \ --master \ --pyargv "--config-file ${DECKHAND_CONFIG_DIR}/deckhand.conf" \ @@ -53,7 +53,7 @@ if [ "$1" = 'server' ]; then --strict \ --vacuum \ --need-app \ - -w deckhand.cmd + --module deckhand.cmd elif [ "$1" = 'alembic' ]; then exec alembic ${@:2} else diff --git a/etc/deckhand/deckhand.conf.sample b/etc/deckhand/deckhand.conf.sample index 29bb5d52..39680e8a 100644 --- a/etc/deckhand/deckhand.conf.sample +++ b/etc/deckhand/deckhand.conf.sample @@ -114,7 +114,7 @@ # Format string to use for log messages with context. Used by # oslo_log.formatters.ContextFormatter (string value) -#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s # Format string to use for log messages when context is undefined. Used by # oslo_log.formatters.ContextFormatter (string value) @@ -131,7 +131,7 @@ # Defines the format string for %(user_identity)s that is used in # logging_context_format_string. Used by oslo_log.formatters.ContextFormatter # (string value) -#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s +#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s # List of package logging levels in logger=LEVEL pairs. This option is ignored # if log_config_append is set. (list value) @@ -242,7 +242,7 @@ # Domain name containing project (string value) #project_domain_name = -# ID of the trust to use as a trustee use (string value) +# Trust ID (string value) #trust_id = # User ID (string value) @@ -508,7 +508,7 @@ # Domain name containing project (string value) #project_domain_name = -# ID of the trust to use as a trustee use (string value) +# Trust ID (string value) #trust_id = # Optional domain ID to use with v3 and v2 parameters. It will be used for both @@ -718,6 +718,9 @@ # middleware should parse the headers or not. (boolean value) #enable_proxy_headers_parsing = false +# HTTP basic auth password file. (string value) +#http_basic_auth_user_file = /etc/htpasswd + [oslo_policy] diff --git a/print_sys_path.py b/print_sys_path.py new file mode 100644 index 00000000..67b624ee --- /dev/null +++ b/print_sys_path.py @@ -0,0 +1,4 @@ +import sys +import pprint + +pprint.pprint(sys.path) diff --git a/requirements-direct.txt b/requirements-direct.txt index f21b04cb..93ab76c7 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -1,77 +1,27 @@ -alembic<=1.4.3 -amqp==2.6.1 -Beaker<=1.12.0 -chardet==3.0.4 -charset_normalizer >=2.0.0, < 4.0.0 -ConfigParser -coverage -cryptography==3.4.8 -deepdiff<=5.8.1 -docutils<=0.17 +barbican==14.0.2 +Beaker +deepdiff falcon -hacking -importlib-metadata~=2.0 -Jinja2<=3.0.3 +fixtures jsonpath_ng -jsonpath-rw-ext>=1.0.0 -jsonpickle==1.4.1 -jsonschema<=3.2.0 -keystoneauth1<=5.1.1 -keystonemiddleware -kombu<=4.6.11 -MarkupSafe<2.1.0, >=0.9.2 +jsonpickle +jsonschema +keystoneauth1==4.3.1 networkx -nose -oslo.cache<=2.10.1 -oslo.concurrency -oslo.config<=8.7.1 -oslo.context<=4.1.0 -oslo.db<=10.0.0 -oslo.log<=4.6.0 -oslo.messaging<=12.13.0 -oslo.middleware<=4.4.0 -oslo.policy<=3.10.1 -oslo.serialization<=4.2.0 -oslo.utils<=4.12.3 -packaging==21.3 -Paste<=3.5.0 -PasteDeploy +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.log==4.8.0 +oslo.policy==3.10.1 +oslo.serialization==4.1.0 +oslo.utils==4.12.3 +oslo.db==11.2.0 PasteScript -pbr<=5.5.1 -psycopg2-binary -pycadf<=3.1.1 -pycodestyle<=2.6.0 -pyflakes<=2.2.0 -Pygments<=2.14.0 -pylibyaml==0.1.0 -pymongo -pyparsing<=2.4.7 -pyproject-api -python-barbicanclient<=5.2.0 -python-dateutil==2.8.1 -python-keystoneclient -python-memcached -python-subunit<=1.4.0 -PyYAML<=5.4.1 -reno -requests==2.23.0 -Routes -setuptools<=45.2.0 +pylibyaml +python-barbicanclient +PyYAML six -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -SQLAlchemy<=1.3.20 -stevedore -testrepository -testresources -testscenarios -testtools<=2.5.0 +testtools tiddlyweb -typing-extensions==3.7.4.3 -urllib3 >= 1.21.1, <= 1.25.11 -uWSGI==2.0.21 -virtualenv Werkzeug -wheel -wsgi-intercept>=1.2.2 -yq==3.2.1 +psycopg2-binary +uwsgi \ No newline at end of file diff --git a/requirements-frozen-bionic.txt b/requirements-frozen-bionic.txt deleted file mode 100644 index 8aa45bda..00000000 --- a/requirements-frozen-bionic.txt +++ /dev/null @@ -1,156 +0,0 @@ -alabaster==0.7.13 -alembic==1.4.3 -amqp==2.6.1 -argcomplete==3.0.8 -attrs==22.2.0 -autopage==0.5.1 -Babel==2.11.0 -bcrypt==4.0.1 -Beaker==1.12.0 -cachetools==4.2.4 -certifi==2023.5.7 -cffi==1.15.1 -chardet==3.0.4 -charset-normalizer==3.0.1 -cliff==3.10.1 -cmd2==2.4.3 -configparser==5.2.0 -coverage==6.2 -cryptography==3.4.8 -dataclasses==0.8 -debtcollector==2.5.0 -decorator==4.4.2 -deepdiff==5.7.0 -distlib==0.3.6 -dnspython==2.2.1 -docutils==0.17 -dogpile.cache==1.2.0 -dulwich==0.20.50 -eventlet==0.33.3 -extras==1.0.0 -falcon==3.1.1 -fasteners==0.18 -filelock==3.4.1 -fixtures==3.0.0 -flake8==3.8.4 -futurist==2.4.1 -greenlet==2.0.2 -hacking==4.1.0 -html5lib==0.9999999 -httpexceptor==1.4.0 -idna==2.10 -imagesize==1.4.1 -importlib-metadata==2.1.3 -importlib-resources==5.4.0 -iso8601==1.1.0 -Jinja2==3.0.3 -jsonpath-ng==1.5.3 -jsonpath-rw==1.4.0 -jsonpath-rw-ext==1.2.2 -jsonpickle==1.4.1 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==9.5.0 -kombu==4.6.11 -Mako==1.1.6 -MarkupSafe==2.0.1 -mccabe==0.6.1 -msgpack==1.0.5 -netaddr==0.8.0 -netifaces==0.11.0 -networkx==2.5.1 -nose==1.3.7 -ordered-set==4.0.2 -os-service-types==1.7.0 -oslo.cache==2.10.1 -oslo.concurrency==4.5.1 -oslo.config==8.7.1 -oslo.context==4.1.0 -oslo.db==10.0.0 -oslo.i18n==5.1.0 -oslo.log==4.6.0 -oslo.messaging==12.13.0 -oslo.metrics==0.4.0 -oslo.middleware==4.4.0 -oslo.policy==3.10.1 -oslo.serialization==4.2.0 -oslo.service==2.8.0 -oslo.utils==4.12.3 -packaging==21.3 -Paste==3.5.0 -PasteDeploy==2.1.1 -PasteScript==3.3.0 -pbr==5.5.1 -pip==21.3.1 -platformdirs==2.4.0 -ply==3.11 -prettytable==2.5.0 -prometheus-client==0.16.0 -psycopg2-binary==2.9.6 -pycadf==3.1.1 -pycodestyle==2.6.0 -pycparser==2.21 -pyflakes==2.2.0 -Pygments==2.14.0 -pylibyaml==0.1.0 -pymongo==4.1.1 -pyparsing==2.4.7 -pyperclip==1.8.2 -pyproject-api==0.1.0 -pyrsistent==0.18.0 -python-barbicanclient==5.2.0 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-keystoneclient==4.5.0 -python-memcached==1.59 -python-mimeparse==1.6.0 -python-subunit==1.4.0 -pytz==2023.3 -PyYAML==5.4.1 -reno==4.0.0 -repoze.lru==0.7 -requests==2.23.0 -resolver==0.2.1 -rfc3986==1.5.0 -Routes==2.5.1 -selector==0.10.1 -setuptools==45.2.0 -simplejson==3.19.1 -six==1.16.0 -snowballstemmer==2.2.0 -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==1.3.20 -sqlalchemy-migrate==0.13.0 -sqlparse==0.4.4 -statsd==4.0.1 -stevedore==3.5.2 -Tempita==0.5.2 -testrepository==0.0.20 -testresources==2.0.1 -testscenarios==0.5.0 -testtools==2.5.0 -tiddlyweb==2.4.3 -tomli==1.2.3 -tomlkit==0.11.6 -typing-extensions==3.7.4.3 -urllib3==1.25.11 -uWSGI==2.0.21 -vine==1.3.0 -virtualenv==20.16.2 -wcwidth==0.2.6 -WebOb==1.8.7 -Werkzeug==2.0.3 -wheel==0.37.1 -wrapt==1.15.0 -wsgi-intercept==1.11.0 -xmltodict==0.13.0 -yappi==1.4.0 -yq==3.2.1 -zipp==3.6.0 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 436313cf..9c11945e 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,154 +1,123 @@ -alabaster==0.7.13 -alembic==1.4.3 -amqp==2.6.1 -argcomplete==3.0.8 +alembic==1.11.1 +amqp==5.1.1 attrs==23.1.0 autopage==0.5.1 -Babel==2.12.1 +backports.zoneinfo==0.2.1 +barbican==14.0.2 bcrypt==4.0.1 -Beaker==1.12.0 -cachetools==5.3.0 -certifi==2022.12.7 +Beaker==1.12.1 +cachetools==5.3.1 +castellan==4.1.0 +certifi==2023.5.7 cffi==1.15.1 -chardet==3.0.4 charset-normalizer==3.1.0 -cliff==3.10.1 +cliff==4.3.0 cmd2==2.4.3 -configparser==5.3.0 -coverage==7.2.3 -cryptography==3.4.8 +cryptography==40.0.2 debtcollector==2.5.0 decorator==5.1.1 -deepdiff==5.8.1 -distlib==0.3.6 +deepdiff==6.3.1 dnspython==2.3.0 -docutils==0.17 -dogpile.cache==1.1.8 -dulwich==0.21.3 +dogpile.cache==1.2.1 eventlet==0.33.3 -extras==1.0.0 falcon==3.1.1 fasteners==0.18 -filelock==3.12.0 -fixtures==3.0.0 -flake8==3.8.4 +fixtures==4.1.0 futurist==2.4.1 greenlet==2.0.2 -hacking==4.1.0 html5lib==0.9999999 httpexceptor==1.4.0 -idna==2.10 -imagesize==1.4.1 -importlib-metadata==2.1.3 -iso8601==1.1.0 -Jinja2==3.0.3 +idna==3.4 +importlib-metadata==4.13.0 +importlib-resources==5.12.0 +iso8601==2.0.0 +Jinja2==3.1.2 jsonpath-ng==1.5.3 -jsonpath-rw==1.4.0 -jsonpath-rw-ext==1.2.2 -jsonpickle==1.4.1 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==10.2.0 -kombu==4.6.11 +jsonpickle==3.0.1 +jsonschema==4.17.3 +keystoneauth1==4.3.1 +keystonemiddleware==10.4.0 +kombu==5.3.0 +ldap3==2.9.1 +logutils==0.3.5 Mako==1.2.4 -MarkupSafe==2.0.1 -mccabe==0.6.1 +MarkupSafe==2.1.3 msgpack==1.0.5 netaddr==0.8.0 netifaces==0.11.0 networkx==3.1 -nose==1.3.7 ordered-set==4.1.0 os-service-types==1.7.0 -oslo.cache==2.10.1 +oslo.cache==3.4.0 oslo.concurrency==5.1.1 -oslo.config==8.7.1 -oslo.context==4.1.0 -oslo.db==10.0.0 +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.db==11.2.0 oslo.i18n==6.0.0 -oslo.log==4.6.0 -oslo.messaging==12.13.0 +oslo.log==4.8.0 +oslo.messaging==14.3.1 oslo.metrics==0.6.0 -oslo.middleware==4.4.0 +oslo.middleware==5.1.1 oslo.policy==3.10.1 -oslo.serialization==4.2.0 +oslo.serialization==4.1.0 oslo.service==3.1.1 +oslo.upgradecheck==2.1.1 oslo.utils==4.12.3 +oslo.versionedobjects==3.1.0 packaging==21.3 -Paste==3.5.0 +Paste==3.5.3 PasteDeploy==3.0.1 PasteScript==3.3.0 -pbr==5.5.1 -pip==23.0.1 -platformdirs==3.2.0 +pbr==5.11.1 +pecan==1.4.2 +pip==23.1.2 +pkgutil_resolve_name==1.3.10 ply==3.11 -prettytable==3.7.0 -prometheus-client==0.16.0 +prettytable==3.8.0 +prometheus-client==0.17.0 psycopg2-binary==2.9.6 +pyasn1==0.4.8 pycadf==3.1.1 -pycodestyle==2.6.0 pycparser==2.21 -pyflakes==2.2.0 -Pygments==2.14.0 pylibyaml==0.1.0 -pymongo==4.3.3 -pyparsing==2.4.7 +pyOpenSSL==23.2.0 +pyparsing==3.0.9 pyperclip==1.8.2 -pyproject_api==1.5.0 pyrsistent==0.19.3 -python-barbicanclient==5.2.0 -python-dateutil==2.8.1 -python-editor==1.0.4 +python-barbicanclient==5.4.0 +python-dateutil==2.8.2 python-keystoneclient==5.1.0 -python-memcached==1.59 python-mimeparse==1.6.0 -python-subunit==1.4.0 pytz==2023.3 -PyYAML==5.4.1 -reno==4.0.0 +PyYAML==6.0 repoze.lru==0.7 -requests==2.23.0 +requests==2.31.0 resolver==0.2.1 -rfc3986==2.0.0 +rfc3986==1.5.0 Routes==2.5.1 selector==0.10.1 -setuptools==45.2.0 +setuptools==67.7.2 simplejson==3.19.1 six==1.16.0 -snowballstemmer==2.2.0 -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==1.3.20 +SQLAlchemy==1.4.48 sqlalchemy-migrate==0.13.0 sqlparse==0.4.4 statsd==4.0.1 -stevedore==5.0.0 +stevedore==5.1.0 Tempita==0.5.2 -testrepository==0.0.20 testresources==2.0.1 testscenarios==0.5.0 -testtools==2.5.0 +testtools==2.6.0 tiddlyweb==2.4.3 -tomli==2.0.1 -tomlkit==0.11.7 -typing-extensions==3.7.4.3 -urllib3==1.25.11 +typing_extensions==4.6.3 +tzdata==2023.3 +urllib3==1.26.16 uWSGI==2.0.21 -vine==1.3.0 -virtualenv==20.22.0 +vine==5.0.0 wcwidth==0.2.6 WebOb==1.8.7 -Werkzeug==2.1.2 +Werkzeug==2.2.3 wheel==0.40.0 wrapt==1.15.0 -wsgi-intercept==1.11.0 -xmltodict==0.13.0 yappi==1.4.0 -yq==3.2.1 zipp==3.15.0 diff --git a/test-requirements.txt b/test-requirements.txt index 4d573831..a664112d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,29 +1,29 @@ -amqp +# amqp bandit==1.6.0 -chardet==3.0.4 -fixtures==3.0.0 +# chardet==3.0.4 +# fixtures==3.0.0 flake8==3.8.4 gabbi==1.35.1 -jsonpickle==1.4.1 +# jsonpickle==1.4.1 +# openstacksdk==0.36.5 openstacksdk==0.36.5 -openstacksdk==0.36.5 -os-testr==2.0.1 +# os-testr==2.0.1 os-testr==2.0.1 oslotest==4.5.0 -packaging==21.3 +# packaging==21.3 pifpaf==3.1.5 -pylibyaml==0.1.0 -pyparsing==2.4.7 +# pylibyaml==0.1.0 +# pyparsing==2.4.7 pytest >= 3.0 -pytest-cov==4.0.0 +pytest-cov python-subunit==1.4.0 -requests==2.23.0 -six==1.16.0 +# requests==2.23.0 +# six==1.16.0 stestr==3.2.0 testrepository==0.0.20 -testtools==2.5.0 +# testtools==2.5.0 tox>=3.28.0, <4.0.0 -urllib3==1.25.11 -yq==3.2.1 \ No newline at end of file +# urllib3==1.25.11 +# yq==3.2.1 \ No newline at end of file diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index 5abd1384..063cf69a 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -16,6 +16,9 @@ roles: - clear-firewall - bindep + - ensure-docker + - ensure-python + - ensure-pip - disable-systemd-resolved - install-test-requirements @@ -46,11 +49,7 @@ set -ex export DISTRO={{ DECKHAND_IMAGE_DISTRO }} make images - if test "${DISTRO}" = 'ubuntu_bionic' - then - # this trick is needed to use bionic image instead of focal in airskiff deployment test - docker tag quay.io/airshipit/deckhand:latest-ubuntu_bionic quay.io/airshipit/deckhand:latest-ubuntu_focal - fi + docker system prune --force args: chdir: "{{ zuul.project.src_dir }}" become: yes @@ -81,6 +80,7 @@ - name: Wait for deployment completion shell: | + set -x ./tools/gate/wait-for-shipyard.sh args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" diff --git a/tools/gate/playbooks/airskiff-reduce-site.yaml b/tools/gate/playbooks/airskiff-reduce-site.yaml index 7c80ed77..2fe6f372 100644 --- a/tools/gate/playbooks/airskiff-reduce-site.yaml +++ b/tools/gate/playbooks/airskiff-reduce-site.yaml @@ -16,6 +16,7 @@ tasks: - name: Overwrite Armada manifest shell: | + set -ex git checkout v1.9 mv tools/gate/manifests/full-site.yaml \ type/skiff/manifests/full-site.yaml diff --git a/tools/gate/playbooks/git-config.yaml b/tools/gate/playbooks/git-config.yaml index d4a118e5..5c044f70 100644 --- a/tools/gate/playbooks/git-config.yaml +++ b/tools/gate/playbooks/git-config.yaml @@ -16,7 +16,7 @@ tasks: - name: Git config shell: | - set -xe; + set -xe tee .git/config << EOF [remote "origin"] url = https://opendev.org/airship/treasuremap.git diff --git a/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml b/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml index 309ad7ea..759d1da3 100644 --- a/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml +++ b/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml @@ -12,28 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Set modprobe br_netfilter - shell: | - set -xe; - sudo modprobe br_netfilter - sudo sysctl net.bridge.bridge-nf-call-iptables=1 - sudo sysctl net.bridge.bridge-nf-call-ip6tables=1 - - name: Install pip3 and gabbi shell: | set -xe; - apt-get install -y \ - python3-pip \ - python3-setuptools \ - python3-dev - pip3 install --upgrade pip - pip3 install -r test-requirements.txt - if test "{{ distro }}" = 'ubuntu_bionic' - then - pip3 install -r requirements-frozen-bionic.txt - else - pip3 install -r requirements-frozen.txt - fi + pip3 install \ + -r test-requirements.txt \ + -r requirements-frozen.txt args: chdir: "{{ zuul.project.src_dir }}" become: yes diff --git a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml index 7df8c180..ed04b52c 100644 --- a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +++ b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml @@ -55,13 +55,10 @@ # NOTE(felipemonteiro): We don't use a venv because they don't play nicely # with OpenStack-Helm, which is used to orchestrate various OpenStack # services in the integration script called below. - sudo -H -E pip3 install -e . --verbose --use-pep517 - if test "{{ distro }}" = 'ubuntu_bionic' - then - sudo -E -H pip3 install -r requirements-frozen-bionic.txt - else - sudo -E -H pip3 install -r requirements-frozen.txt - fi + sudo -E -H pip3 install -e . --verbose --use-pep517 + sudo -E -H pip3 install -r requirements-frozen.txt + echo "$(pwd)" | sudo tee /usr/local/lib/python3.8/dist-packages/deckhand.pth + pifpaf run postgresql -- ./tools/integration-tests.sh args: chdir: "{{ zuul.project.src_dir }}" diff --git a/tools/gate/scripts/020-deploy-postgresql.sh b/tools/gate/scripts/020-deploy-postgresql.sh index a902842a..3afe2ce7 100755 --- a/tools/gate/scripts/020-deploy-postgresql.sh +++ b/tools/gate/scripts/020-deploy-postgresql.sh @@ -23,7 +23,7 @@ POSTGRES_ID=$( -e POSTGRES_DB=deckhand \ -e POSTGRES_USER=deckhand \ -e POSTGRES_PASSWORD=password \ - postgres:14.6 + postgres:14.8 ) POSTGRES_IP=$( diff --git a/tools/integration-tests.sh b/tools/integration-tests.sh index 5a0718cc..34a74656 100755 --- a/tools/integration-tests.sh +++ b/tools/integration-tests.sh @@ -13,8 +13,17 @@ set -xe CURRENT_DIR="$(pwd)" -: ${OSH_INFRA_PATH:="../openstack-helm-infra"} -: ${OSH_PATH:="../openstack-helm"} +: "${OSH_INFRA_PATH:="../openstack-helm-infra"}" +: "${OSH_PATH:="../openstack-helm"}" +: "${TM_PATH:="../treasuremap"}" + +export MAKE_CHARTS_OPENSTACK_HELM="${MAKE_CHARTS_OPENSTACK_HELM:-true}" +export MAKE_CHARTS_OSH_INFRA="${MAKE_CHARTS_OSH_INFRA:-true}" +export MAKE_CHARTS_ARMADA="${MAKE_CHARTS_ARMADA:-false}" +export MAKE_CHARTS_DECKHAND="${MAKE_CHARTS_DECKHAND:-false}" +export MAKE_CHARTS_SHIPYARD="${MAKE_CHARTS_SHIPYARD:-false}" +export MAKE_CHARTS_MAAS="${MAKE_CHARTS_MAAS:-false}" +export MAKE_CHARTS_PORTHOLE="${MAKE_CHARTS_PORTHOLE:-false}" function deploy_barbican { @@ -46,34 +55,58 @@ function deploy_osh_keystone_barbican { git clone https://git.openstack.org/openstack/openstack-helm.git ../openstack-helm fi - cd ${OSH_INFRA_PATH} - # git reset --hard ${BARBICAN_STABLE_COMMIT} + if [ ! -d "$TM_PATH" ]; then + git clone https://git.openstack.org/airship/treasuremap.git ../treasuremap + pushd ../treasuremap + git checkout v1.9 + popd + fi + + cd "${TM_PATH}" # Deploy required packages - ./tools/deployment/common/000-install-packages.sh - ./tools/deployment/common/001-setup-apparmor-profiles.sh - # - cd ${OSH_PATH} - # git reset --hard ${BARBICAN_STABLE_COMMIT} - # Deploy required packages - ./tools/deployment/common/install-packages.sh + ./tools/deployment/airskiff/developer/009-setup-apparmor.sh # # Deploy Kubernetes - sudo modprobe br_netfilter - ./tools/deployment/common/deploy-k8s.sh + ./tools/deployment/airskiff/developer/010-deploy-k8s.sh + # + # Make charts + ./tools/deployment/airskiff/developer/015-make-all-charts.sh + # - cd ${CURRENT_DIR} + cd "${OSH_PATH}" + # Deploy docker-based openstack client + if [ -f /usr/bin/openstack ] ||[ -f /usr/local/bin/openstack ] + then + sudo rm -rf /usr/bin/openstack /usr/local/bin/openstack + fi + sudo -H mkdir -p /etc/openstack + sudo -H chown -R $(id -un): /etc/openstack + tee /etc/openstack/clouds.yaml << EOF + clouds: + openstack_helm: + region_name: RegionOne + identity_api_version: 3 + auth: + username: 'admin' + password: 'password' + project_name: 'admin' + project_domain_name: 'default' + user_domain_name: 'default' + auth_url: 'http://keystone.openstack.svc.cluster.local/v3' +EOF + sudo tee /usr/local/bin/openstack << EOF + #!/bin/bash - # remove systemd-resolved local stub dns from resolv.conf - sudo sed -i.bkp '/^nameserver.*127.0.0.1/d - w /dev/stdout' /etc/resolv.conf - # add external nameservers - echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf - echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf - cat /etc/resolv.conf + docker run \ + -v /etc/openstack/clouds.yaml:/etc/openstack/clouds.yaml \ + -e OS_CLOUD=\${OS_CLOUD} \ + quay.io/airshipit/porthole-openstack-utility:latest-ubuntu_focal \ + openstack \ + --os-cloud=\${OS_CLOUD} \ + \$@ +EOF + sudo chmod +x /usr/local/bin/openstack - cd ${OSH_PATH} - # Setup clients on the host and assemble the charts - ./tools/deployment/common/setup-client.sh # Deploy the ingress controller ./tools/deployment/component/common/ingress.sh # Deploy NFS Provisioner @@ -98,11 +131,12 @@ function deploy_deckhand { cd ${CURRENT_DIR} + # TODO(fmontei): Use Keystone bootstrap override instead. interfaces=("admin" "public" "internal") deckhand_endpoint="http://127.0.0.1:9000" - if [ -z "$( openstack service list --format value 2>/dev/null | grep deckhand )" ]; then + if [ -z "$( openstack_client openstack service list --format value 2>/dev/null | grep deckhand )" ]; then openstack service create --enable --name deckhand deckhand 2>/dev/null fi diff --git a/tox.ini b/tox.ini index 833d9811..e5d07524 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,7 @@ passenv = deps = -r{toxinidir}/requirements-frozen.txt -r{toxinidir}/test-requirements.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt commands = find . -type f -name "*.pyc" -delete rm -Rf .testrepository/times.dbm @@ -53,37 +54,12 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt commands= rm -f requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt" -[testenv:freeze-bionic] -basepython=python3 -recreate = True -allowlist_externals= - rm - sh -deps= - -r{toxinidir}/requirements-direct.txt -commands= - rm -f requirements-frozen-bionic.txt - sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt" - -[testenv:py36] -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -allowlist_externals = - stestr - find - rm -commands = - {[testenv]commands} - stestr run {posargs} - stestr slowest - - [testenv:py38] allowlist_externals = stestr @@ -103,35 +79,8 @@ allowlist_externals = find rm -[testenv:py36-postgresql] -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -commands = - {[testenv]commands} - bash {toxinidir}/tools/run_pifpaf.sh '{posargs}' -allowlist_externals = - bash - find - rm - -[testenv:py36-functional] -basepython=python3 -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -allowlist_externals = - find - stestr - pip3 - echo -setenv = VIRTUAL_ENV={envdir} -commands = - find . -type f -name "*.pyc" -delete - stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs} - [testenv:py38-functional] -basepython=python3 +basepython=python3.8 allowlist_externals = find stestr @@ -144,7 +93,7 @@ commands = [testenv:py38-functional-dev] -basepython=python3 +basepython=python3.8 # Minimalistic functional test job for running Deckhand functional tests # via uwsgi. Uses pifpaf for DB instantiation. Useful for developers. # Requires PostgreSQL be installed on host. @@ -159,33 +108,11 @@ commands = find . -type f -name "*.pyc" -delete pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}" -[testenv:py36-functional-dev] -basepython=python3 -# Minimalistic functional test job for running Deckhand functional tests -# via uwsgi. Uses pifpaf for DB instantiation. Useful for developers. -# Requires PostgreSQL be installed on host. -deps = - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -allowlist_externals = - find - stestr - pifpaf - pip3 -setenv = VIRTUAL_ENV={envdir} -commands = - pip3 install -e . --verbose --use-pep517 - find . -type f -name "*.pyc" -delete - pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}" - [testenv:cover] allowlist_externals = find stestr coverage -deps = - -r{toxinidir}/requirements-frozen.txt - -r{toxinidir}/test-requirements.txt setenv = {[testenv]setenv} PYTHON=coverage run --source deckhand --parallel-mode commands =