Airflow stable 2.6.2

This PS updates python modules and code to match Airflow 2.6.2

Change-Id: If6f57325339995216d2553c7a5ff56e7673b5acc

.zuul.yaml

@@ -14,31 +14,29 @@
     templates:
       - openstack-cover-jobs
       - openstack-python38-jobs
-      - openstack-python36-jobs
+
     check:
       jobs:
         - openstack-tox-docs
         - openstack-tox-pep8
         - armada-chart-build-gate
         - armada-chart-build-latest-htk
-        - armada-docker-build-gate-ubuntu_bionic
         - armada-docker-build-gate-ubuntu_focal
         - armada-airskiff-deploy-focal
-        - armada-airskiff-deploy-bionic
+
     gate:
       jobs:
         - openstack-tox-docs
         - openstack-tox-pep8
         - armada-chart-build-gate
-        - armada-docker-build-gate-ubuntu_bionic
         - armada-docker-build-gate-ubuntu_focal
         - armada-airskiff-deploy-focal
-        - armada-airskiff-deploy-bionic
+
     post:
       jobs:
         - armada-docker-publish-ubuntu_focal
-        - armada-docker-publish-ubuntu_bionic
         - armada-upload-git-mirror
+
     promote:
       jobs:
         - promote-airship-project-docs
@@ -77,22 +75,9 @@
     vars:
       HTK_COMMIT: master
 
-- job:
-    name: armada-docker-build-gate-ubuntu_bionic
-    timeout: 1800
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: armada-single-node
-    vars:
-      publish: false
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          patch_set: true
-
-
 - job:
     name: armada-docker-build-gate-ubuntu_focal
-    timeout: 1800
+    timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: armada-single-node-focal
     vars:
@@ -118,7 +103,7 @@
       - airship/treasuremap
     vars:
       CLONE_ARMADA: false
-      OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
+      OSH_INFRA_COMMIT: 3d8935a536faaa87e084e7783ed53f4660a3a1f8
       ARMADA_IMAGE_DISTRO: ubuntu_focal
     irrelevant-files:
       - ^.*\.rst$
@@ -127,37 +112,9 @@
       - ^releasenotes/.*$
       - ^swagger/.*$
 
-
-- job:
-    name: armada-airskiff-deploy-bionic
-    nodeset: armada-single-node-focal
-    description: |
-      Deploy Memcached using Airskiff and submitted Armada changes.
-    timeout: 9600
-    voting: true
-    pre-run:
-      - tools/gate/playbooks/airskiff-reduce-site.yaml
-      - tools/gate/playbooks/git-config.yaml
-    run: tools/gate/playbooks/airskiff-deploy.yaml
-    post-run: tools/gate/playbooks/debug-report.yaml
-    required-projects:
-      - airship/treasuremap
-    vars:
-      CLONE_ARMADA: false
-      OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
-      ARMADA_IMAGE_DISTRO: ubuntu_bionic
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^doc/.*$
-      - ^examples/.*$
-      - ^releasenotes/.*$
-      - ^swagger/.*$
-
-
-
 - job:
     name: armada-docker-publish-ubuntu_focal
-    timeout: 1800
+    timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: armada-single-node-focal
     secrets:
@@ -172,22 +129,6 @@
         static:
           - latest
 
-- job:
-    name: armada-docker-publish-ubuntu_bionic
-    timeout: 1800
-    run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: armada-single-node
-    secrets:
-      - airship_armada_quay_creds
-    vars:
-      publish: true
-      distro: ubuntu_bionic
-      tags:
-        dynamic:
-          branch: true
-          commit: true
-        static:
-          - latest
 
 - secret:
     name: airship_armada_quay_creds

armada/handlers/metrics.py

@@ -115,13 +115,15 @@ class ActionWithTimeoutMetrics(ActionMetrics):
 
         timeout.observe(timeout_value)
 
-        def observe_timeout_usage(duration):
+        class observe_timeout_usage:
-            # Avoid division by 0
+            def calculate_timeout_usage(duration):
-            if timeout_value:
+                # Avoid division by 0
-                val = duration / timeout_value
+                if timeout_value:
-                timeout_usage.observe(val)
+                    val = duration / timeout_value
+                    timeout_usage.observe(val)
 
-        timer = context_managers.Timer(observe_timeout_usage)
+        timer = context_managers.Timer(
+            observe_timeout_usage, 'calculate_timeout_usage')
         context = super().get_context(*args, **kwargs)
         context.enter_context(timer)
         return context

armada/tests/unit/handlers/test_override.py

@@ -273,7 +273,7 @@ class OverrideTestCase(testtools.TestCase):
             ovr = Override(documents)
             ovr.update_documents(merging_values)
             ovr_doc = ovr.find_manifest_document(doc_path)
-            expect_doc = list(yaml.load_all(e.read()))[0]
+            expect_doc = list(yaml.safe_load_all(e.read()))[0]
 
             self.assertEqual(ovr_doc, expect_doc)
 
@@ -290,7 +290,7 @@ class OverrideTestCase(testtools.TestCase):
             ovr = Override(documents, override)
             ovr.update_manifests()
             ovr_doc = ovr.find_manifest_document(doc_path)
-            target_docs = list(yaml.load_all(e.read()))
+            target_docs = list(yaml.safe_load_all(e.read()))
             expected_doc = [
                 x for x in target_docs
                 if x.get('schema') == 'armada/Manifest/v1'

doc/requirements.txt

@@ -3,9 +3,7 @@
 # process, which may cause wedges in the gate later.
 docutils<=0.17
 Jinja2<=3.0.3
-MarkupSafe<2.1.0, >=0.9.2
 plantuml
-pylibyaml==0.1.0
 reno
 sphinx-rtd-theme==0.5.0
 Sphinx==3.3.1

doc/source/_static/armada.conf.sample

@@ -152,7 +152,7 @@
 
 # Format string to use for log messages with context. Used by
 # oslo_log.formatters.ContextFormatter (string value)
-#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
+#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s
 
 # Format string to use for log messages when context is undefined. Used by
 # oslo_log.formatters.ContextFormatter (string value)
@@ -169,7 +169,7 @@
 # Defines the format string for %(user_identity)s that is used in
 # logging_context_format_string. Used by oslo_log.formatters.ContextFormatter
 # (string value)
-#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
+#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s
 
 # List of package logging levels in logger=LEVEL pairs. This option is ignored
 # if log_config_append is set. (list value)
@@ -261,7 +261,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>
 
 # Optional domain ID to use with v3 and v2 parameters. It will be used for both

etc/armada/armada.conf.sample

@@ -152,7 +152,7 @@
 
 # Format string to use for log messages with context. Used by
 # oslo_log.formatters.ContextFormatter (string value)
-#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
+#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s
 
 # Format string to use for log messages when context is undefined. Used by
 # oslo_log.formatters.ContextFormatter (string value)
@@ -169,7 +169,7 @@
 # Defines the format string for %(user_identity)s that is used in
 # logging_context_format_string. Used by oslo_log.formatters.ContextFormatter
 # (string value)
-#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
+#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s
 
 # List of package logging levels in logger=LEVEL pairs. This option is ignored
 # if log_config_append is set. (list value)
@@ -321,7 +321,7 @@
 # Domain name containing project (string value)
 #project_domain_name = <None>
 
-# ID of the trust to use as a trustee use (string value)
+# Trust ID (string value)
 #trust_id = <None>
 
 # Optional domain ID to use with v3 and v2 parameters. It will be used for both

images/armada/Dockerfile.ubuntu_focal

@@ -94,6 +94,8 @@ ENV PBR_VERSION 0.8.0
 
 RUN \
     chown -R armada:users . && \
-    python3 setup.py install
+    pip3 install -e . --verbose --use-pep517 \
+    && echo "/armada" \
+        > /usr/local/lib/python3.8/dist-packages/deckhand.pth
 
 USER armada

requirements-direct.txt

@@ -1,36 +1,31 @@
 click
-configparser
+deepdiff
-deepdiff<=5.8.1
-docutils<=0.17
 falcon
-gitpython
+GitPython
-importlib-metadata~=2.0
+jsonschema
-Jinja2<=3.0.3
+keystoneauth1==4.3.1
-jsonschema<=3.2.0
+keystonemiddleware==9.3.0
-keystoneauth1<=5.1.1
+kubernetes
-kubernetes==26.1.0
+mock
-MarkupSafe<2.1.0, >=0.9.2
+oslo.config==8.6.0
-nose
+oslo.context==4.0.0
-oslo.config<=8.7.1
+oslo.log==4.8.0
-oslo.i18n
+oslo.policy==3.10.1
-oslo.log<=4.6.0
+oslo.middleware==4.4.0
-oslo.policy<=3.10.1
+oslo.serialization==4.1.0
-oslo.middleware<=4.4.0
+oslo.utils==4.12.3
-keystonemiddleware<=10.2.0
+oslo.db==11.2.0
-oslo.utils<=4.12.3
+paste
-Paste>=2.0.3
+PasteDeploy
-PasteDeploy>=1.5.2
+packaging
-prometheus-client==0.8.0
+pbr
-pylibyaml==0.1.0
+prometheus-client
-python-dateutil==2.8.1
+pylibyaml
-PyYAML<=5.4.1
+python_dateutil
-reno
+PyYAML
-requests==2.23.0
+Requests
+responses
 retry
-setuptools<=45.2.0
+testtools
-six
+urllib3
-Sphinx==3.3.1
+uWSGI
-sphinx_rtd_theme==0.5.0
-urllib3 >= 1.21.1, <= 1.25.11
-uWSGI==2.0.21
-wheel

requirements-frozen-bionic.txt

@@ -1,92 +0,0 @@
-alabaster==0.7.13
-attrs==22.2.0
-Babel==2.11.0
-bcrypt==4.0.1
-cachetools==4.2.4
-certifi==2023.5.7
-chardet==3.0.4
-click==8.0.4
-configparser==5.2.0
-debtcollector==2.5.0
-decorator==5.1.1
-deepdiff==5.7.0
-docutils==0.17
-dogpile.cache==1.2.0
-dulwich==0.20.50
-falcon==3.1.1
-gitdb==4.0.9
-GitPython==3.1.18
-google-auth==2.18.1
-idna==2.10
-imagesize==1.4.1
-importlib-metadata==2.1.3
-importlib-resources==5.4.0
-iso8601==1.1.0
-Jinja2==3.0.3
-jsonschema==3.2.0
-keystoneauth1==5.1.1
-keystonemiddleware==9.5.0
-kubernetes==26.1.0
-MarkupSafe==2.0.1
-msgpack==1.0.5
-netaddr==0.8.0
-netifaces==0.11.0
-nose==1.3.7
-oauthlib==3.2.2
-ordered-set==4.0.2
-os-service-types==1.7.0
-oslo.cache==2.11.0
-oslo.config==8.7.1
-oslo.context==4.1.0
-oslo.i18n==5.1.0
-oslo.log==4.6.0
-oslo.middleware==4.4.0
-oslo.policy==3.10.1
-oslo.serialization==4.3.0
-oslo.utils==4.12.3
-packaging==21.3
-Paste==3.5.3
-PasteDeploy==2.1.1
-pbr==5.11.1
-pip==21.3.1
-prometheus-client==0.8.0
-py==1.11.0
-pyasn1==0.5.0
-pyasn1-modules==0.3.0
-pycadf==3.1.1
-Pygments==2.14.0
-pylibyaml==0.1.0
-pyparsing==3.0.9
-pyrsistent==0.18.0
-python-dateutil==2.8.1
-python-keystoneclient==4.5.0
-pytz==2023.3
-PyYAML==5.4.1
-reno==4.0.0
-requests==2.23.0
-requests-oauthlib==1.3.1
-retry==0.9.2
-rfc3986==1.5.0
-rsa==4.9
-setuptools==45.2.0
-six==1.16.0
-smmap==5.0.0
-snowballstemmer==2.2.0
-Sphinx==3.3.1
-sphinx-rtd-theme==0.5.0
-sphinxcontrib-applehelp==1.0.2
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.0
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-statsd==4.0.1
-stevedore==3.5.2
-typing_extensions==4.1.1
-urllib3==1.25.11
-uWSGI==2.0.21
-WebOb==1.8.7
-websocket-client==1.3.1
-wheel==0.37.1
-wrapt==1.15.0
-zipp==3.6.0

requirements-frozen.txt

@@ -1,90 +1,90 @@
-alabaster==0.7.13
+alembic==1.11.1
 attrs==23.1.0
-Babel==2.12.1
 bcrypt==4.0.1
-cachetools==5.3.0
+cachetools==5.3.1
-certifi==2022.12.7
+certifi==2023.5.7
-chardet==3.0.4
+charset-normalizer==3.1.0
 click==8.1.3
-configparser==5.3.0
 debtcollector==2.5.0
 decorator==5.1.1
-deepdiff==5.8.1
+deepdiff==6.3.1
-docutils==0.17
+dogpile.cache==1.2.2
-dogpile.cache==1.2.0
-dulwich==0.21.3
 falcon==3.1.1
+fixtures==4.1.0
 gitdb==4.0.10
 GitPython==3.1.31
-google-auth==2.17.3
+google-auth==2.20.0
-idna==2.10
+greenlet==2.0.2
-imagesize==1.4.1
+idna==3.4
-importlib-metadata==2.1.3
+importlib-metadata==4.13.0
-iso8601==1.1.0
+importlib-resources==5.12.0
-Jinja2==3.0.3
+iso8601==2.0.0
-jsonschema==3.2.0
+Jinja2==3.1.2
-keystoneauth1==5.1.1
+jsonschema==4.17.3
-keystonemiddleware==10.2.0
+keystoneauth1==4.3.1
-kubernetes==26.1.0
+keystonemiddleware==9.3.0
-MarkupSafe==2.0.1
+kubernetes==23.6.0
+Mako==1.2.4
+MarkupSafe==2.1.3
+mock==5.1.0
 msgpack==1.0.5
 netaddr==0.8.0
 netifaces==0.11.0
-nose==1.3.7
 oauthlib==3.2.2
 ordered-set==4.1.0
 os-service-types==1.7.0
-oslo.cache==3.3.1
+oslo.cache==3.4.0
-oslo.config==8.7.1
+oslo.config==8.6.0
-oslo.context==5.1.1
+oslo.context==4.0.0
+oslo.db==11.2.0
 oslo.i18n==6.0.0
-oslo.log==4.6.0
+oslo.log==4.8.0
 oslo.middleware==4.4.0
 oslo.policy==3.10.1
-oslo.serialization==5.1.1
+oslo.serialization==4.1.0
 oslo.utils==4.12.3
-packaging==23.1
+packaging==21.3
 Paste==3.5.3
 PasteDeploy==3.0.1
 pbr==5.11.1
-pip==23.0.1
+pip==23.1.2
-prometheus-client==0.8.0
+pkgutil_resolve_name==1.3.10
+prometheus-client==0.17.0
 py==1.11.0
-pyasn1==0.5.0
+pyasn1==0.4.8
-pyasn1-modules==0.3.0
+pyasn1-modules==0.2.8
 pycadf==3.1.1
-Pygments==2.15.1
 pylibyaml==0.1.0
 pyparsing==3.0.9
 pyrsistent==0.19.3
-python-dateutil==2.8.1
+python-dateutil==2.8.2
 python-keystoneclient==5.1.0
 pytz==2023.3
-PyYAML==5.4.1
+PyYAML==6.0
-reno==4.0.0
+requests==2.31.0
-requests==2.23.0
 requests-oauthlib==1.3.1
+responses==0.23.1
 retry==0.9.2
-rfc3986==2.0.0
+rfc3986==1.5.0
 rsa==4.9
-setuptools==45.2.0
+setuptools==67.7.2
 six==1.16.0
 smmap==5.0.0
-snowballstemmer==2.2.0
+SQLAlchemy==1.4.48
-Sphinx==3.3.1
+sqlalchemy-migrate==0.13.0
-sphinx-rtd-theme==0.5.0
+sqlparse==0.4.4
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
 statsd==4.0.1
-stevedore==5.0.0
+stevedore==5.1.0
-urllib3==1.25.11
+Tempita==0.5.2
+testresources==2.0.1
+testscenarios==0.5.0
+testtools==2.6.0
+types-PyYAML==6.0.12.10
+typing_extensions==4.6.3
+urllib3==1.26.16
 uWSGI==2.0.21
 WebOb==1.8.7
-websocket-client==1.5.1
+websocket-client==1.5.3
 wheel==0.40.0
 wrapt==1.15.0
 zipp==3.15.0

test-requirements.txt

@@ -1,20 +1,17 @@
 bandit==1.6.0
-cmd2>=1.5.0
+# cmd2>=1.5.0
 coverage==5.3
 flake8-import-order>=0.18.1
 flake8==3.8.4
 grpcio-tools>=1.16.0
-mock==4.0.2
 os-testr>=1.0.0 # Apache-2.0
 pyflakes==2.2.0
-responses==0.12.1
 sphinx-rtd-theme==0.5.0
 stestr==3.2.0
 testrepository==0.0.20
 testresources==2.0.1
 testscenarios==0.5.0
-testtools==2.5.0
+# testtools==2.5.0
-testtools==2.5.0
+# tox>=3.28.0, <4.0.0
-tox>=3.28.0, <4.0.0
+# typing-extensions>=3.7.2
-typing-extensions>=3.7.2
 yapf==0.27.0

tools/gate/playbooks/airskiff-deploy.yaml

@@ -20,6 +20,9 @@
     - ensure-python
     - ensure-pip
     - ensure-docker
+    - role: add-authorized-keys
+      public_keys:
+        - public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
 
   tasks:
 
@@ -48,11 +51,7 @@
         set -ex
         export DISTRO={{ ARMADA_IMAGE_DISTRO }}
         make images
-        if test "${DISTRO}" = 'ubuntu_bionic'
+        docker system prune --force
-        then
-          # this trick is needed to use bionic image instead of focal in airskiff deployment test
-          docker tag quay.io/airshipit/armada:latest-ubuntu_bionic quay.io/airshipit/armada:latest-ubuntu_focal
-        fi
       args:
         chdir: "{{ zuul.project.src_dir }}"
       become: yes
@@ -65,10 +64,21 @@
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
       become: yes
 
+    - name: Setup openstack client
+      shell: |
+        ./tools/deployment/airskiff/developer/020-setup-client.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Deploy Airship components using Armada
       shell: |
+        set -x
         mkdir -p ~/.kube
         cp -rp /home/zuul/.kube/config ~/.kube/config
+
+        while true; do sleep 100; done
+
         ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"

tools/gate/playbooks/docker-image-build.yaml

@@ -18,7 +18,6 @@
     - ensure-docker
     - ensure-python
     - ensure-pip
-    - ensure-tox
 
   tasks:
     - include_vars: vars.yaml
@@ -62,6 +61,8 @@
     - name: Run images
       when: not publish
       shell: |
+        set -ex
+        sudo -E -H pip3 install tox==3.28.0
         make run_images
       args:
         chdir: "{{ zuul.project.src_dir }}"

tox.ini

@@ -30,7 +30,6 @@ commands =
 commands =
   {posargs}
 
-
 [testenv:freeze]
 basepython=python3
 recreate = True
@@ -39,33 +38,12 @@ allowlist_externals=
   sh
 deps=
   -r{toxinidir}/requirements-direct.txt
+  -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
 commands=
   rm -f requirements-frozen.txt
   sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt"
 
 
-[testenv:freeze-bionic]
-basepython=python3
-recreate = True
-allowlist_externals=
-  rm
-  sh
-deps=
-  -r{toxinidir}/requirements-direct.txt
-commands=
-  rm -f requirements-frozen-bionic.txt
-  sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt"
-
-
-[testenv:py36]
-deps=
-    -r{toxinidir}/requirements-frozen-bionic.txt
-    -r{toxinidir}/test-requirements.txt
-commands =
-    {[testenv]commands}
-    stestr run {posargs}
-    stestr slowest
-
 [testenv:py38]
 commands =
     {[testenv]commands}