Airflow stable 2.6.2

This PS updates python modules and code to match Airflow 2.6.2

Change-Id: I65a1b86473ee3e988aae353b59fb5473d75851f9

.zuul.yaml

@@ -12,57 +12,43 @@
 
 - project:
     templates:
-      - openstack-cover-jobs
       - docs-on-readthedocs
       - openstack-python38-jobs
-      - openstack-python36-jobs
     vars:
       rtd_webhook_id: '38572'
       rtd_project_name: 'airship-deckhand'
     check:
       jobs:
-        - deckhand-tox-py36-postgresql
         - deckhand-tox-py38-postgresql
-        - deckhand-functional-uwsgi-py36
         - deckhand-functional-uwsgi-py38
-        - deckhand-functional-docker-py38-ubuntu_focal
+        - deckhand-functional-docker-py38
-        - deckhand-functional-docker-py36-ubuntu_bionic
-        - deckhand-integration-uwsgi-py36
         - deckhand-integration-uwsgi-py38
-        - deckhand-integration-docker-py38-ubuntu_focal
+        - deckhand-integration-docker-py38
-        - deckhand-integration-docker-py36-ubuntu_bionic
         - deckhand-chart-build-gate
         - deckhand-chart-build-latest-htk
-        - deckhand-docker-build-gate-ubuntu_focal
+        - deckhand-docker-build-gate
-        - deckhand-docker-build-gate-ubuntu_bionic
+        - deckhand-openstack-tox-pep8-focal
-        - openstack-tox-pep8
+        - deckhand-openstack-tox-cover-focal
         - deckhand-airskiff-deployment-focal
-        - deckhand-airskiff-deployment-bionic
+
     gate:
       jobs:
-        - deckhand-tox-py36-postgresql
         - deckhand-tox-py38-postgresql
-        - deckhand-functional-uwsgi-py36
         - deckhand-functional-uwsgi-py38
-        - deckhand-functional-docker-py38-ubuntu_focal
+        - deckhand-functional-docker-py38
-        - deckhand-functional-docker-py36-ubuntu_bionic
-        - deckhand-integration-uwsgi-py36
         - deckhand-integration-uwsgi-py38
-        - deckhand-integration-docker-py38-ubuntu_focal
+        - deckhand-integration-docker-py38
-        - deckhand-integration-docker-py36-ubuntu_bionic
         - deckhand-chart-build-gate
-        - deckhand-docker-build-gate-ubuntu_focal
+        - deckhand-docker-build-gate
-        - deckhand-docker-build-gate-ubuntu_bionic
+        - deckhand-openstack-tox-pep8-focal
-        - openstack-tox-pep8
+        - deckhand-openstack-tox-cover-focal
         - deckhand-airskiff-deployment-focal
-        - deckhand-airskiff-deployment-bionic
+
     post:
       jobs:
         - deckhand-upload-git-mirror
         - deckhand-docker-publish-ubuntu_focal
         - deckhand-docker-tag-ubuntu_focal
-        - deckhand-docker-publish-ubuntu_bionic
-        - deckhand-docker-tag-ubuntu_bionic
 
 
 - nodeset:
@@ -89,6 +75,18 @@
       - name: primary
         label: ubuntu-focal
 
+- job:
+    name: deckhand-openstack-tox-pep8-focal
+    parent: openstack-tox-pep8
+    description: Runs pep8 job on focal
+    nodeset: deckhand-single-node-airskiff-focal
+
+- job:
+    name: deckhand-openstack-tox-cover-focal
+    parent: openstack-tox-cover
+    description: Runs cover job on focal
+    nodeset: deckhand-single-node-airskiff-focal
+
 - job:
     name: deckhand-tox-py38-postgresql
     parent: openstack-tox-py38
@@ -98,35 +96,6 @@
     vars:
       tox_envlist: py38-postgresql
 
-- job:
-    name: deckhand-tox-py36-postgresql
-    parent: openstack-tox-py38
-    nodeset: deckhand-single-node
-    pre-run:
-      - tools/gate/playbooks/install-postgresql.yaml
-    vars:
-      tox_envlist: py36-postgresql
-
-- job:
-    name: deckhand-functional-uwsgi-py36
-    voting: true
-    description: |
-      Run tox-based functional tests for the Airship Deckhand project using a
-      minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
-      for ephemeral PostgreSQL DB, under cPython version 3.6.
-    run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml
-    nodeset: deckhand-single-node
-    vars:
-      tox_envlist: py36-functional-dev
-      distro: ubuntu_bionic
-    irrelevant-files: &irrelevant-files
-      - ^.*\.rst$
-      - ^doc/.*$
-      - ^etc/.*$
-      - ^releasenotes/.*$
-      - ^setup.cfg$
-      - ^deckhand/tests/unit/.*$
-
 
 - job:
     name: deckhand-functional-uwsgi-py38
@@ -140,7 +109,13 @@
     vars:
       tox_envlist: py38-functional-dev
       distro: ubuntu_focal
-    irrelevant-files: *irrelevant-files
+    irrelevant-files: &irrelevant-files
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+      - ^deckhand/tests/unit/.*$
 
 - job:
     name: deckhand-functional-docker-base
@@ -159,7 +134,7 @@
       - openstack/openstack-helm-infra
 
 - job:
-    name: deckhand-functional-docker-py38-ubuntu_focal
+    name: deckhand-functional-docker-py38
     description: |
       Run tox-based functional tests for the Airship Deckhand project under
       cPython version 3.8. Uses tox with the ``functional-py38`` environment.
@@ -172,36 +147,6 @@
       distro: ubuntu_focal
     irrelevant-files: *irrelevant-files
 
-- job:
-    name: deckhand-functional-docker-py36-ubuntu_bionic
-    description: |
-      Run tox-based functional tests for the Airship Deckhand project under
-      cPython version 3.6. Uses tox with the ``functional-py36`` environment.
-      Ubuntu (focal) image is built and used.
-    parent: deckhand-functional-docker-base
-    nodeset: deckhand-single-node
-    vars:
-      tox_envlist: py36-functional
-      disable_keystone: true
-      distro: ubuntu_bionic
-    irrelevant-files: *irrelevant-files
-
-
-
-- job:
-    name: deckhand-integration-uwsgi-py36
-    voting: true
-    description: |
-      Run tox-based integration tests for the Airship Deckhand project using a
-      minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
-      for ephemeral PostgreSQL DB, under cPython version 3.6.
-    timeout: 3600
-    run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml
-    nodeset: deckhand-single-node
-    irrelevant-files: *irrelevant-files
-    vars:
-      disable_keystone: true
-      distro: ubuntu_bionic
 
 - job:
     name: deckhand-integration-uwsgi-py38
@@ -210,7 +155,7 @@
       Run tox-based integration tests for the Airship Deckhand project using a
       minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
       for ephemeral PostgreSQL DB, under cPython version 3.8.
-    timeout: 3600
+    timeout: 9600
     run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml
     nodeset: deckhand-single-node-focal
     irrelevant-files: *irrelevant-files
@@ -263,7 +208,7 @@
       - ^releasenotes/.*$
 
 - job:
-    name: deckhand-integration-docker-py38-ubuntu_focal
+    name: deckhand-integration-docker-py38
     description: |
       Run tox-based integration tests for the Airship Deckhand project under
       cPython version 3.8. Builds ubuntu (focal) deckhand image.
@@ -273,17 +218,6 @@
       disable_keystone: false
       distro: ubuntu_focal
 
-- job:
-    name: deckhand-integration-docker-py36-ubuntu_bionic
-    description: |
-      Run tox-based integration tests for the Airship Deckhand project under
-      cPython version 3.6. Builds ubuntu (focal) deckhand image.
-    parent: deckhand-integration-docker-base
-    nodeset: openstack-helm-single-node
-    vars:
-      disable_keystone: false
-      distro: ubuntu_focal
-
 - job:
     name: deckhand-airskiff-deployment-focal
     voting: true
@@ -292,15 +226,15 @@
       Deploy Memcached using Airskiff and submitted Deckhand changes.
     timeout: 9600
     pre-run:
-      - tools/gate/playbooks/airskiff-reduce-site.yaml
       - tools/gate/playbooks/git-config.yaml
+      - tools/gate/playbooks/airskiff-reduce-site.yaml
     run: tools/gate/playbooks/airskiff-deploy.yaml
     post-run: tools/gate/playbooks/debug-report.yaml
     required-projects:
       - airship/treasuremap
     vars:
       CLONE_DECKHAND: false
-      OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
+      OSH_INFRA_COMMIT: 3d8935a536faaa87e084e7783ed53f4660a3a1f8
       DECKHAND_IMAGE_DISTRO: ubuntu_focal
       distro: ubuntu_focal
     irrelevant-files:
@@ -308,33 +242,9 @@
       - ^doc/.*$
       - ^releasenotes/.*$
 
-- job:
-    name: deckhand-airskiff-deployment-bionic
-    voting: true
-    nodeset: deckhand-single-node-airskiff-focal
-    description: |
-      Deploy Memcached using Airskiff and submitted Deckhand changes.
-    timeout: 9600
-    pre-run:
-      - tools/gate/playbooks/airskiff-reduce-site.yaml
-      - tools/gate/playbooks/git-config.yaml
-    run: tools/gate/playbooks/airskiff-deploy.yaml
-    post-run: tools/gate/playbooks/debug-report.yaml
-    required-projects:
-      - airship/treasuremap
-    vars:
-      CLONE_DECKHAND: false
-      OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
-      DECKHAND_IMAGE_DISTRO: ubuntu_bionic
-      distro: ubuntu_focal
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^doc/.*$
-      - ^releasenotes/.*$
-
 
 - job:
-    name: deckhand-docker-build-gate-ubuntu_focal
+    name: deckhand-docker-build-gate
     timeout: 1800
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: deckhand-single-node-focal
@@ -352,19 +262,6 @@
         dynamic:
           patch_set: true
 
-- job:
-    name: deckhand-docker-build-gate-ubuntu_bionic
-    timeout: 1800
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: deckhand-single-node
-    irrelevant-files: *non-code-files-template
-    vars:
-      publish: false
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          patch_set: true
-
 - job:
     name: deckhand-docker-publish-ubuntu_focal
     description: |
@@ -387,28 +284,6 @@
         static:
           - latest
 
-- job:
-    name: deckhand-docker-publish-ubuntu_bionic
-    description: |
-      Runs on every merge, unless files in a dictionary below are changed.
-      Builds and publishes container ubuntu images on quay.io with a set of tags
-      listed in vars section. Waits in Zuul queue for a node (VM) assignment.
-    timeout: 1800
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: deckhand-single-node
-    secrets:
-      - airship_deckhand_quay_creds
-    irrelevant-files: *non-code-files-template
-    vars:
-      publish: true
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          branch: true
-          commit: true
-        static:
-          - latest
-
 - job:
     name: deckhand-docker-tag-ubuntu_focal
     description: |
@@ -425,23 +300,6 @@
     vars:
       distro: ubuntu_focal
 
-- job:
-    name: deckhand-docker-tag-ubuntu_bionic
-    description: |
-      Runs on every merge when files in a dictionalry below are changed, and
-      adds git commit id tag onto the ubuntu container image published on quay.io,
-      which has `latest` tag set. Does not wait in queue for a node (VM)
-      assignment, runs almost immediately.
-    timeout: 1800
-    run: tools/gate/playbooks/docker-image-tag.yaml
-    nodeset:
-      nodes: []
-    secrets:
-      - airship_deckhand_quay_creds
-    vars:
-      distro: ubuntu_bionic
-
-
 
 - secret:
     name: airship_deckhand_quay_creds

ChangeLog

@@ -1,6 +1,9 @@
 CHANGES
 =======
 
+* Airflow stable 2.6.2
+* Restored ubuntu\_bionic image build
+* [focal] Fix requests.body attribute deprecation
 * Update airskiff deployment gate
 * Deckhand updates
 * Removing egg-info folder

charts/deckhand/values.yaml

@@ -29,7 +29,7 @@ images:
   tags:
     deckhand: quay.io/airshipit/deckhand:latestlatest-ubuntu_focal
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
-    db_init: docker.io/postgres:14.6
+    db_init: docker.io/postgres:14.8
     db_sync: quay.io/airshipit/deckhand:latest-ubuntu_focal
     image_repo_sync: docker.io/docker:23.0.3
     ks_endpoints: docker.io/openstackhelm/heat:wallaby-ubuntu_focal

deckhand/conf/config.py

@@ -43,7 +43,7 @@ barbican_opts = [
                 help="Whether to enable Barbican secret caching. Useful "
                      "for testing to avoid cross-test caching conflicts."),
     cfg.StrOpt(
-        'cache_timeout', default=3600,
+        'cache_timeout', default='3600',
         help="How long (in seconds) Barbican secret reference/payload lookup "
              "results should remain cached in memory.")
 ]
@@ -63,7 +63,7 @@ engine_opts = [
     cfg.BoolOpt('enable_cache', default=True,
                 help="Whether to enable the document rendering caching. Useful"
                      " for testing to avoid cross-test caching conflicts."),
-    cfg.IntOpt('cache_timeout', default=3600,
+    cfg.IntOpt('cache_timeout', default='3600',
                help="How long (in seconds) document rendering results should "
                     "remain cached in memory."),
 ]
@@ -76,7 +76,7 @@ jsonpath_group = cfg.OptGroup(
 
 
 jsonpath_opts = [
-    cfg.IntOpt('cache_timeout', default=3600,
+    cfg.IntOpt('cache_timeout', default='3600',
                help="How long (in seconds) JSONPath lookup results should "
                     "remain cached in memory.")
 ]

deckhand/tests/common/test_gabbi.py

@@ -90,7 +90,7 @@ class MultidocJsonpaths(jsonhandler.JSONHandler):
         # NOTE: The simple approach to handling dictionary versus list response
         # bodies is to always parse the response body as a list and index into
         # the first element using [0] throughout the tests.
-        return list(yaml.load_all(string))
+        return list(yaml.safe_load_all(string))
 
 
 def load_tests(loader, tests, pattern):

entrypoint.sh

@@ -38,13 +38,13 @@ echo "Command: $1 with arguments $@"
 # Start deckhand application
 if [ "$1" = 'server' ]; then
     exec uwsgi \
-        -b 32768 \
+        --buffer-size 32768 \
         --callable deckhand_callable \
         --die-on-term \
         --enable-threads \
         --http :${PORT} \
         --http-timeout ${DECKHAND_API_TIMEOUT} \
-        -L \
+        --disable-logging \
         --lazy-apps \
         --master \
         --pyargv "--config-file ${DECKHAND_CONFIG_DIR}/deckhand.conf" \
@@ -53,7 +53,7 @@ if [ "$1" = 'server' ]; then
         --strict \
         --vacuum \
         --need-app \
-        -w deckhand.cmd
+        --module deckhand.cmd
 elif [ "$1" = 'alembic' ]; then
     exec alembic ${@:2}
 else

etc/deckhand/deckhand.conf.sample

@@ -114,7 +114,7 @@
 
 # Format string to use for log messages with context. Used by
 # oslo_log.formatters.ContextFormatter (string value)
-#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
+#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s
 
 # Format string to use for log messages when context is undefined. Used by
 # oslo_log.formatters.ContextFormatter (string value)
@@ -131,7 +131,7 @@
 # Defines the format string for %(user_identity)s that is used in
 # logging_context_format_string. Used by oslo_log.formatters.ContextFormatter
 # (string value)
-#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
+#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s
 
 # List of package logging levels in logger=LEVEL pairs. This option is ignored
 # if log_config_append is set. (list value)
@@ -242,7 +242,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>
 
 # User ID (string value)
@@ -508,7 +508,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>
 
 # Optional domain ID to use with v3 and v2 parameters. It will be used for both
@@ -718,6 +718,9 @@
 # middleware should parse the headers or not. (boolean value)
 #enable_proxy_headers_parsing = false
 
+# HTTP basic auth password file. (string value)
+#http_basic_auth_user_file = /etc/htpasswd
+
 
 [oslo_policy]
 

print_sys_path.py

@@ -0,0 +1,4 @@
+import sys
+import pprint
+
+pprint.pprint(sys.path)

requirements-direct.txt

@@ -1,77 +1,27 @@
-alembic<=1.4.3
+barbican==14.0.2
-amqp==2.6.1
+Beaker
-Beaker<=1.12.0
+deepdiff
-chardet==3.0.4
-charset_normalizer >=2.0.0, < 4.0.0
-ConfigParser
-coverage
-cryptography==3.4.8
-deepdiff<=5.8.1
-docutils<=0.17
 falcon
-hacking
+fixtures
-importlib-metadata~=2.0
-Jinja2<=3.0.3
 jsonpath_ng
-jsonpath-rw-ext>=1.0.0
+jsonpickle
-jsonpickle==1.4.1
+jsonschema
-jsonschema<=3.2.0
+keystoneauth1==4.3.1
-keystoneauth1<=5.1.1
-keystonemiddleware
-kombu<=4.6.11
-MarkupSafe<2.1.0, >=0.9.2
 networkx
-nose
+oslo.config==8.6.0
-oslo.cache<=2.10.1
+oslo.context==4.0.0
-oslo.concurrency
+oslo.log==4.8.0
-oslo.config<=8.7.1
+oslo.policy==3.10.1
-oslo.context<=4.1.0
+oslo.serialization==4.1.0
-oslo.db<=10.0.0
+oslo.utils==4.12.3
-oslo.log<=4.6.0
+oslo.db==11.2.0
-oslo.messaging<=12.13.0
-oslo.middleware<=4.4.0
-oslo.policy<=3.10.1
-oslo.serialization<=4.2.0
-oslo.utils<=4.12.3
-packaging==21.3
-Paste<=3.5.0
-PasteDeploy
 PasteScript
-pbr<=5.5.1
+pylibyaml
-psycopg2-binary
+python-barbicanclient
-pycadf<=3.1.1
+PyYAML
-pycodestyle<=2.6.0
-pyflakes<=2.2.0
-Pygments<=2.14.0
-pylibyaml==0.1.0
-pymongo
-pyparsing<=2.4.7
-pyproject-api
-python-barbicanclient<=5.2.0
-python-dateutil==2.8.1
-python-keystoneclient
-python-memcached
-python-subunit<=1.4.0
-PyYAML<=5.4.1
-reno
-requests==2.23.0
-Routes
-setuptools<=45.2.0
 six
-Sphinx==3.3.1
+testtools
-sphinx-rtd-theme==0.5.0
-SQLAlchemy<=1.3.20
-stevedore
-testrepository
-testresources
-testscenarios
-testtools<=2.5.0
 tiddlyweb
-typing-extensions==3.7.4.3
-urllib3 >= 1.21.1, <= 1.25.11
-uWSGI==2.0.21
-virtualenv
 Werkzeug
-wheel
+psycopg2-binary
-wsgi-intercept>=1.2.2
+uwsgi
-yq==3.2.1

requirements-frozen-bionic.txt

@@ -1,156 +0,0 @@
-alabaster==0.7.13
-alembic==1.4.3
-amqp==2.6.1
-argcomplete==3.0.8
-attrs==22.2.0
-autopage==0.5.1
-Babel==2.11.0
-bcrypt==4.0.1
-Beaker==1.12.0
-cachetools==4.2.4
-certifi==2023.5.7
-cffi==1.15.1
-chardet==3.0.4
-charset-normalizer==3.0.1
-cliff==3.10.1
-cmd2==2.4.3
-configparser==5.2.0
-coverage==6.2
-cryptography==3.4.8
-dataclasses==0.8
-debtcollector==2.5.0
-decorator==4.4.2
-deepdiff==5.7.0
-distlib==0.3.6
-dnspython==2.2.1
-docutils==0.17
-dogpile.cache==1.2.0
-dulwich==0.20.50
-eventlet==0.33.3
-extras==1.0.0
-falcon==3.1.1
-fasteners==0.18
-filelock==3.4.1
-fixtures==3.0.0
-flake8==3.8.4
-futurist==2.4.1
-greenlet==2.0.2
-hacking==4.1.0
-html5lib==0.9999999
-httpexceptor==1.4.0
-idna==2.10
-imagesize==1.4.1
-importlib-metadata==2.1.3
-importlib-resources==5.4.0
-iso8601==1.1.0
-Jinja2==3.0.3
-jsonpath-ng==1.5.3
-jsonpath-rw==1.4.0
-jsonpath-rw-ext==1.2.2
-jsonpickle==1.4.1
-jsonschema==3.2.0
-keystoneauth1==5.1.1
-keystonemiddleware==9.5.0
-kombu==4.6.11
-Mako==1.1.6
-MarkupSafe==2.0.1
-mccabe==0.6.1
-msgpack==1.0.5
-netaddr==0.8.0
-netifaces==0.11.0
-networkx==2.5.1
-nose==1.3.7
-ordered-set==4.0.2
-os-service-types==1.7.0
-oslo.cache==2.10.1
-oslo.concurrency==4.5.1
-oslo.config==8.7.1
-oslo.context==4.1.0
-oslo.db==10.0.0
-oslo.i18n==5.1.0
-oslo.log==4.6.0
-oslo.messaging==12.13.0
-oslo.metrics==0.4.0
-oslo.middleware==4.4.0
-oslo.policy==3.10.1
-oslo.serialization==4.2.0
-oslo.service==2.8.0
-oslo.utils==4.12.3
-packaging==21.3
-Paste==3.5.0
-PasteDeploy==2.1.1
-PasteScript==3.3.0
-pbr==5.5.1
-pip==21.3.1
-platformdirs==2.4.0
-ply==3.11
-prettytable==2.5.0
-prometheus-client==0.16.0
-psycopg2-binary==2.9.6
-pycadf==3.1.1
-pycodestyle==2.6.0
-pycparser==2.21
-pyflakes==2.2.0
-Pygments==2.14.0
-pylibyaml==0.1.0
-pymongo==4.1.1
-pyparsing==2.4.7
-pyperclip==1.8.2
-pyproject-api==0.1.0
-pyrsistent==0.18.0
-python-barbicanclient==5.2.0
-python-dateutil==2.8.1
-python-editor==1.0.4
-python-keystoneclient==4.5.0
-python-memcached==1.59
-python-mimeparse==1.6.0
-python-subunit==1.4.0
-pytz==2023.3
-PyYAML==5.4.1
-reno==4.0.0
-repoze.lru==0.7
-requests==2.23.0
-resolver==0.2.1
-rfc3986==1.5.0
-Routes==2.5.1
-selector==0.10.1
-setuptools==45.2.0
-simplejson==3.19.1
-six==1.16.0
-snowballstemmer==2.2.0
-Sphinx==3.3.1
-sphinx-rtd-theme==0.5.0
-sphinxcontrib-applehelp==1.0.2
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.0
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.20
-sqlalchemy-migrate==0.13.0
-sqlparse==0.4.4
-statsd==4.0.1
-stevedore==3.5.2
-Tempita==0.5.2
-testrepository==0.0.20
-testresources==2.0.1
-testscenarios==0.5.0
-testtools==2.5.0
-tiddlyweb==2.4.3
-tomli==1.2.3
-tomlkit==0.11.6
-typing-extensions==3.7.4.3
-urllib3==1.25.11
-uWSGI==2.0.21
-vine==1.3.0
-virtualenv==20.16.2
-wcwidth==0.2.6
-WebOb==1.8.7
-Werkzeug==2.0.3
-wheel==0.37.1
-wrapt==1.15.0
-wsgi-intercept==1.11.0
-xmltodict==0.13.0
-yappi==1.4.0
-yq==3.2.1
-zipp==3.6.0

requirements-frozen.txt

@@ -1,154 +1,123 @@
-alabaster==0.7.13
+alembic==1.11.1
-alembic==1.4.3
+amqp==5.1.1
-amqp==2.6.1
-argcomplete==3.0.8
 attrs==23.1.0
 autopage==0.5.1
-Babel==2.12.1
+backports.zoneinfo==0.2.1
+barbican==14.0.2
 bcrypt==4.0.1
-Beaker==1.12.0
+Beaker==1.12.1
-cachetools==5.3.0
+cachetools==5.3.1
-certifi==2022.12.7
+castellan==4.1.0
+certifi==2023.5.7
 cffi==1.15.1
-chardet==3.0.4
 charset-normalizer==3.1.0
-cliff==3.10.1
+cliff==4.3.0
 cmd2==2.4.3
-configparser==5.3.0
+cryptography==40.0.2
-coverage==7.2.3
-cryptography==3.4.8
 debtcollector==2.5.0
 decorator==5.1.1
-deepdiff==5.8.1
+deepdiff==6.3.1
-distlib==0.3.6
 dnspython==2.3.0
-docutils==0.17
+dogpile.cache==1.2.1
-dogpile.cache==1.1.8
-dulwich==0.21.3
 eventlet==0.33.3
-extras==1.0.0
 falcon==3.1.1
 fasteners==0.18
-filelock==3.12.0
+fixtures==4.1.0
-fixtures==3.0.0
-flake8==3.8.4
 futurist==2.4.1
 greenlet==2.0.2
-hacking==4.1.0
 html5lib==0.9999999
 httpexceptor==1.4.0
-idna==2.10
+idna==3.4
-imagesize==1.4.1
+importlib-metadata==4.13.0
-importlib-metadata==2.1.3
+importlib-resources==5.12.0
-iso8601==1.1.0
+iso8601==2.0.0
-Jinja2==3.0.3
+Jinja2==3.1.2
 jsonpath-ng==1.5.3
-jsonpath-rw==1.4.0
+jsonpickle==3.0.1
-jsonpath-rw-ext==1.2.2
+jsonschema==4.17.3
-jsonpickle==1.4.1
+keystoneauth1==4.3.1
-jsonschema==3.2.0
+keystonemiddleware==10.4.0
-keystoneauth1==5.1.1
+kombu==5.3.0
-keystonemiddleware==10.2.0
+ldap3==2.9.1
-kombu==4.6.11
+logutils==0.3.5
 Mako==1.2.4
-MarkupSafe==2.0.1
+MarkupSafe==2.1.3
-mccabe==0.6.1
 msgpack==1.0.5
 netaddr==0.8.0
 netifaces==0.11.0
 networkx==3.1
-nose==1.3.7
 ordered-set==4.1.0
 os-service-types==1.7.0
-oslo.cache==2.10.1
+oslo.cache==3.4.0
 oslo.concurrency==5.1.1
-oslo.config==8.7.1
+oslo.config==8.6.0
-oslo.context==4.1.0
+oslo.context==4.0.0
-oslo.db==10.0.0
+oslo.db==11.2.0
 oslo.i18n==6.0.0
-oslo.log==4.6.0
+oslo.log==4.8.0
-oslo.messaging==12.13.0
+oslo.messaging==14.3.1
 oslo.metrics==0.6.0
-oslo.middleware==4.4.0
+oslo.middleware==5.1.1
 oslo.policy==3.10.1
-oslo.serialization==4.2.0
+oslo.serialization==4.1.0
 oslo.service==3.1.1
+oslo.upgradecheck==2.1.1
 oslo.utils==4.12.3
+oslo.versionedobjects==3.1.0
 packaging==21.3
-Paste==3.5.0
+Paste==3.5.3
 PasteDeploy==3.0.1
 PasteScript==3.3.0
-pbr==5.5.1
+pbr==5.11.1
-pip==23.0.1
+pecan==1.4.2
-platformdirs==3.2.0
+pip==23.1.2
+pkgutil_resolve_name==1.3.10
 ply==3.11
-prettytable==3.7.0
+prettytable==3.8.0
-prometheus-client==0.16.0
+prometheus-client==0.17.0
 psycopg2-binary==2.9.6
+pyasn1==0.4.8
 pycadf==3.1.1
-pycodestyle==2.6.0
 pycparser==2.21
-pyflakes==2.2.0
-Pygments==2.14.0
 pylibyaml==0.1.0
-pymongo==4.3.3
+pyOpenSSL==23.2.0
-pyparsing==2.4.7
+pyparsing==3.0.9
 pyperclip==1.8.2
-pyproject_api==1.5.0
 pyrsistent==0.19.3
-python-barbicanclient==5.2.0
+python-barbicanclient==5.4.0
-python-dateutil==2.8.1
+python-dateutil==2.8.2
-python-editor==1.0.4
 python-keystoneclient==5.1.0
-python-memcached==1.59
 python-mimeparse==1.6.0
-python-subunit==1.4.0
 pytz==2023.3
-PyYAML==5.4.1
+PyYAML==6.0
-reno==4.0.0
 repoze.lru==0.7
-requests==2.23.0
+requests==2.31.0
 resolver==0.2.1
-rfc3986==2.0.0
+rfc3986==1.5.0
 Routes==2.5.1
 selector==0.10.1
-setuptools==45.2.0
+setuptools==67.7.2
 simplejson==3.19.1
 six==1.16.0
-snowballstemmer==2.2.0
+SQLAlchemy==1.4.48
-Sphinx==3.3.1
-sphinx-rtd-theme==0.5.0
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-SQLAlchemy==1.3.20
 sqlalchemy-migrate==0.13.0
 sqlparse==0.4.4
 statsd==4.0.1
-stevedore==5.0.0
+stevedore==5.1.0
 Tempita==0.5.2
-testrepository==0.0.20
 testresources==2.0.1
 testscenarios==0.5.0
-testtools==2.5.0
+testtools==2.6.0
 tiddlyweb==2.4.3
-tomli==2.0.1
+typing_extensions==4.6.3
-tomlkit==0.11.7
+tzdata==2023.3
-typing-extensions==3.7.4.3
+urllib3==1.26.16
-urllib3==1.25.11
 uWSGI==2.0.21
-vine==1.3.0
+vine==5.0.0
-virtualenv==20.22.0
 wcwidth==0.2.6
 WebOb==1.8.7
-Werkzeug==2.1.2
+Werkzeug==2.2.3
 wheel==0.40.0
 wrapt==1.15.0
-wsgi-intercept==1.11.0
-xmltodict==0.13.0
 yappi==1.4.0
-yq==3.2.1
 zipp==3.15.0

test-requirements.txt

@@ -1,29 +1,29 @@
 
 
-amqp
+# amqp
 bandit==1.6.0
-chardet==3.0.4
+# chardet==3.0.4
-fixtures==3.0.0
+# fixtures==3.0.0
 flake8==3.8.4
 gabbi==1.35.1
-jsonpickle==1.4.1
+# jsonpickle==1.4.1
+# openstacksdk==0.36.5
 openstacksdk==0.36.5
-openstacksdk==0.36.5
+# os-testr==2.0.1
-os-testr==2.0.1
 os-testr==2.0.1
 oslotest==4.5.0
-packaging==21.3
+# packaging==21.3
 pifpaf==3.1.5
-pylibyaml==0.1.0
+# pylibyaml==0.1.0
-pyparsing==2.4.7
+# pyparsing==2.4.7
 pytest >= 3.0
-pytest-cov==4.0.0
+pytest-cov
 python-subunit==1.4.0
-requests==2.23.0
+# requests==2.23.0
-six==1.16.0
+# six==1.16.0
 stestr==3.2.0
 testrepository==0.0.20
-testtools==2.5.0
+# testtools==2.5.0
 tox>=3.28.0, <4.0.0
-urllib3==1.25.11
+# urllib3==1.25.11
-yq==3.2.1
+# yq==3.2.1

tools/gate/playbooks/airskiff-deploy.yaml

@@ -16,6 +16,9 @@
   roles:
     - clear-firewall
     - bindep
+    - ensure-docker
+    - ensure-python
+    - ensure-pip
     - disable-systemd-resolved
     - install-test-requirements
 
@@ -46,11 +49,7 @@
         set -ex
         export DISTRO={{ DECKHAND_IMAGE_DISTRO }}
         make images
-        if test "${DISTRO}" = 'ubuntu_bionic'
+        docker system prune --force
-        then
-          # this trick is needed to use bionic image instead of focal in airskiff deployment test
-          docker tag quay.io/airshipit/deckhand:latest-ubuntu_bionic quay.io/airshipit/deckhand:latest-ubuntu_focal
-        fi
       args:
         chdir: "{{ zuul.project.src_dir }}"
       become: yes
@@ -81,6 +80,7 @@
 
     - name: Wait for deployment completion
       shell: |
+        set -x
         ./tools/gate/wait-for-shipyard.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"

tools/gate/playbooks/airskiff-reduce-site.yaml

@@ -16,6 +16,7 @@
   tasks:
     - name: Overwrite Armada manifest
       shell: |
+        set -ex
         git checkout v1.9
         mv tools/gate/manifests/full-site.yaml \
             type/skiff/manifests/full-site.yaml

tools/gate/playbooks/git-config.yaml

@@ -16,7 +16,7 @@
   tasks:
     - name: Git config
       shell: |
-        set -xe;
+        set -xe
         tee .git/config << EOF
         [remote "origin"]
         url = https://opendev.org/airship/treasuremap.git

tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml

@@ -12,28 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Set modprobe br_netfilter
-  shell: |
-    set -xe;
-    sudo modprobe br_netfilter
-    sudo sysctl net.bridge.bridge-nf-call-iptables=1
-    sudo sysctl net.bridge.bridge-nf-call-ip6tables=1
-
 - name: Install pip3 and gabbi
   shell: |
     set -xe;
-    apt-get install -y \
+    pip3 install \
-      python3-pip \
+      -r test-requirements.txt \
-      python3-setuptools \
+      -r requirements-frozen.txt
-      python3-dev
-    pip3 install --upgrade pip
-    pip3 install -r test-requirements.txt
-    if test "{{ distro }}" = 'ubuntu_bionic'
-    then
-      pip3 install -r requirements-frozen-bionic.txt
-    else
-      pip3 install -r requirements-frozen.txt
-    fi
   args:
     chdir: "{{ zuul.project.src_dir }}"
   become: yes

tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml

@@ -55,13 +55,10 @@
     # NOTE(felipemonteiro): We don't use a venv because they don't play nicely
     # with OpenStack-Helm, which is used to orchestrate various OpenStack
     # services in the integration script called below.
-    sudo -H -E pip3 install -e . --verbose --use-pep517
+    sudo -E -H pip3 install -e . --verbose --use-pep517
-    if test "{{ distro }}" = 'ubuntu_bionic'
-    then
-      sudo -E -H pip3 install -r requirements-frozen-bionic.txt
-    else
     sudo -E -H pip3 install -r requirements-frozen.txt
-    fi
+    echo "$(pwd)" | sudo tee /usr/local/lib/python3.8/dist-packages/deckhand.pth
+
     pifpaf run postgresql -- ./tools/integration-tests.sh
   args:
     chdir: "{{ zuul.project.src_dir }}"

tools/gate/scripts/020-deploy-postgresql.sh

@@ -23,7 +23,7 @@ POSTGRES_ID=$(
         -e POSTGRES_DB=deckhand \
         -e POSTGRES_USER=deckhand \
         -e POSTGRES_PASSWORD=password \
-            postgres:14.6
+            postgres:14.8
 )
 
 POSTGRES_IP=$(

tools/integration-tests.sh

@@ -13,8 +13,17 @@
 set -xe
 
 CURRENT_DIR="$(pwd)"
-: ${OSH_INFRA_PATH:="../openstack-helm-infra"}
+: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
-: ${OSH_PATH:="../openstack-helm"}
+: "${OSH_PATH:="../openstack-helm"}"
+: "${TM_PATH:="../treasuremap"}"
+
+export MAKE_CHARTS_OPENSTACK_HELM="${MAKE_CHARTS_OPENSTACK_HELM:-true}"
+export MAKE_CHARTS_OSH_INFRA="${MAKE_CHARTS_OSH_INFRA:-true}"
+export MAKE_CHARTS_ARMADA="${MAKE_CHARTS_ARMADA:-false}"
+export MAKE_CHARTS_DECKHAND="${MAKE_CHARTS_DECKHAND:-false}"
+export MAKE_CHARTS_SHIPYARD="${MAKE_CHARTS_SHIPYARD:-false}"
+export MAKE_CHARTS_MAAS="${MAKE_CHARTS_MAAS:-false}"
+export MAKE_CHARTS_PORTHOLE="${MAKE_CHARTS_PORTHOLE:-false}"
 
 
 function deploy_barbican {
@@ -46,34 +55,58 @@ function deploy_osh_keystone_barbican {
         git clone https://git.openstack.org/openstack/openstack-helm.git ../openstack-helm
     fi
 
-    cd ${OSH_INFRA_PATH}
+    if [ ! -d "$TM_PATH" ]; then
-    # git reset --hard ${BARBICAN_STABLE_COMMIT}
+        git clone https://git.openstack.org/airship/treasuremap.git ../treasuremap
+        pushd ../treasuremap
+        git checkout v1.9
+        popd
+    fi
+
+    cd "${TM_PATH}"
     # Deploy required packages
-    ./tools/deployment/common/000-install-packages.sh
+    ./tools/deployment/airskiff/developer/009-setup-apparmor.sh
-    ./tools/deployment/common/001-setup-apparmor-profiles.sh
-    #
-    cd ${OSH_PATH}
-    # git reset --hard ${BARBICAN_STABLE_COMMIT}
-    # Deploy required packages
-    ./tools/deployment/common/install-packages.sh
     #
     # Deploy Kubernetes
-    sudo modprobe br_netfilter
+    ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
-    ./tools/deployment/common/deploy-k8s.sh
+    #
+    # Make charts
+    ./tools/deployment/airskiff/developer/015-make-all-charts.sh
+    #
 
-    cd ${CURRENT_DIR}
+    cd "${OSH_PATH}"
+    # Deploy docker-based openstack client
+    if [ -f /usr/bin/openstack ] ||[ -f /usr/local/bin/openstack ]
+    then
+        sudo rm -rf /usr/bin/openstack /usr/local/bin/openstack
+    fi
+    sudo -H mkdir -p /etc/openstack
+    sudo -H chown -R $(id -un): /etc/openstack
+    tee /etc/openstack/clouds.yaml << EOF
+    clouds:
+        openstack_helm:
+        region_name: RegionOne
+        identity_api_version: 3
+        auth:
+            username: 'admin'
+            password: 'password'
+            project_name: 'admin'
+            project_domain_name: 'default'
+            user_domain_name: 'default'
+            auth_url: 'http://keystone.openstack.svc.cluster.local/v3'
+EOF
+    sudo tee /usr/local/bin/openstack << EOF
+    #!/bin/bash
 
-    # remove systemd-resolved local stub dns from resolv.conf
+    docker run \
-    sudo sed -i.bkp '/^nameserver.*127.0.0.1/d
+        -v /etc/openstack/clouds.yaml:/etc/openstack/clouds.yaml \
-                     w /dev/stdout' /etc/resolv.conf
+        -e OS_CLOUD=\${OS_CLOUD} \
-    # add external nameservers
+        quay.io/airshipit/porthole-openstack-utility:latest-ubuntu_focal \
-    echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
+            openstack \
-    echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf
+                --os-cloud=\${OS_CLOUD} \
-    cat /etc/resolv.conf
+                \$@
+EOF
+    sudo chmod +x /usr/local/bin/openstack
 
-    cd ${OSH_PATH}
-    # Setup clients on the host and assemble the charts
-    ./tools/deployment/common/setup-client.sh
     # Deploy the ingress controller
     ./tools/deployment/component/common/ingress.sh
     # Deploy NFS Provisioner
@@ -98,11 +131,12 @@ function deploy_deckhand {
 
     cd ${CURRENT_DIR}
 
+
     # TODO(fmontei): Use Keystone bootstrap override instead.
     interfaces=("admin" "public" "internal")
     deckhand_endpoint="http://127.0.0.1:9000"
 
-    if [ -z "$( openstack service list --format value 2>/dev/null | grep deckhand )" ]; then
+    if [ -z "$( openstack_client openstack service list --format value 2>/dev/null | grep deckhand )" ]; then
         openstack service create --enable --name deckhand deckhand 2>/dev/null
     fi
 

tox.ini

@@ -34,6 +34,7 @@ passenv =
 deps =
   -r{toxinidir}/requirements-frozen.txt
   -r{toxinidir}/test-requirements.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
 commands =
   find . -type f -name "*.pyc" -delete
   rm -Rf .testrepository/times.dbm
@@ -53,37 +54,12 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/requirements-direct.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
 commands=
   rm -f requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"
 
 
-[testenv:freeze-bionic]
-basepython=python3
-recreate = True
-allowlist_externals=
-  rm
-  sh
-deps=
-  -r{toxinidir}/requirements-direct.txt
-commands=
-  rm -f requirements-frozen-bionic.txt
-  sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt"
-
-[testenv:py36]
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-allowlist_externals =
-  stestr
-  find
-  rm
-commands =
-  {[testenv]commands}
-  stestr run {posargs}
-  stestr slowest
-
-
 [testenv:py38]
 allowlist_externals =
   stestr
@@ -103,35 +79,8 @@ allowlist_externals =
   find
   rm
 
-[testenv:py36-postgresql]
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-commands =
-    {[testenv]commands}
-    bash {toxinidir}/tools/run_pifpaf.sh '{posargs}'
-allowlist_externals =
-  bash
-  find
-  rm
-
-[testenv:py36-functional]
-basepython=python3
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-allowlist_externals =
-  find
-  stestr
-  pip3
-  echo
-setenv = VIRTUAL_ENV={envdir}
-commands =
-  find . -type f -name "*.pyc" -delete
-  stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs}
-
 [testenv:py38-functional]
-basepython=python3
+basepython=python3.8
 allowlist_externals =
   find
   stestr
@@ -144,7 +93,7 @@ commands =
 
 
 [testenv:py38-functional-dev]
-basepython=python3
+basepython=python3.8
 # Minimalistic functional test job for running Deckhand functional tests
 # via uwsgi. Uses pifpaf for DB instantiation. Useful for developers.
 # Requires PostgreSQL be installed on host.
@@ -159,33 +108,11 @@ commands =
   find . -type f -name "*.pyc" -delete
   pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}"
 
-[testenv:py36-functional-dev]
-basepython=python3
-# Minimalistic functional test job for running Deckhand functional tests
-# via uwsgi. Uses pifpaf for DB instantiation. Useful for developers.
-# Requires PostgreSQL be installed on host.
-deps =
-  -r{toxinidir}/requirements-frozen-bionic.txt
-  -r{toxinidir}/test-requirements.txt
-allowlist_externals =
-  find
-  stestr
-  pifpaf
-  pip3
-setenv = VIRTUAL_ENV={envdir}
-commands =
-  pip3 install -e . --verbose --use-pep517
-  find . -type f -name "*.pyc" -delete
-  pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}"
-
 [testenv:cover]
 allowlist_externals =
   find
   stestr
   coverage
-deps =
-  -r{toxinidir}/requirements-frozen.txt
-  -r{toxinidir}/test-requirements.txt
 setenv = {[testenv]setenv}
          PYTHON=coverage run --source deckhand --parallel-mode
 commands =