Fix compatibility issues

A lot of things have changed at once (or have been ignored
in refstack for some time) and in order to keep refstack
functioning this commit addressed the following:
* Update nodejs jobs so that we keep testing refstack
with the newest nodejs
* Update jobs for Zed development cycle
* Add support for py3.10
* Fix 2009327 story - refstack is now compatible with
jsonschema>=3.2.0 (schema definition got fixed)
* Drop py3.6 and 3.7 (upstream tooling as e.g. devstack
isn't supporting those anymore)
* Fix mysql setup script used to run functional unit tests
* Add WebTest dependency needed by unit tests
* Update alembic and jsonschema version to at least
currently recomended versions (by constraints)

Task: 43761
Story: 2009327
Change-Id: I37bb7dc520119bdab280d62e23922641ef7658ce

.zuul.yaml

@@ -1,17 +1,15 @@
 - project:
     templates:
-      - nodejs6-jobs
+      - nodejs16-jobs
       - openstack-cover-jobs
-      - openstack-python3-yoga-jobs
+      - openstack-python3-zed-jobs
     check:
       jobs:
-        - refstack-tox-functional:
-            nodeset: openstack-single-node-bionic
+        - refstack-tox-functional
         - opendev-tox-docs
     gate:
       jobs:
-        - refstack-tox-functional:
-            nodeset: openstack-single-node-bionic
+        - refstack-tox-functional
         - opendev-tox-docs
     promote:
       jobs:
@@ -19,7 +17,7 @@
 
 - job:
     name: refstack-tox-functional
-    parent: openstack-tox
+    parent: openstack-tox-with-sudo
     description: |
       Run functional tests for an OpenStack Python project under cPython 3.
       Uses tox with the ``functional`` environment.

refstack/api/validators.py

@@ -98,7 +98,7 @@ class TestResultValidator(BaseValidator):
             'duration_seconds': {'type': 'integer'},
             'results': {
                 'type': 'array',
-                'items': [{
+                'items': {
                     'type': 'object',
                     'properties': {
                         'name': {'type': 'string'},
@@ -107,8 +107,7 @@ class TestResultValidator(BaseValidator):
                             'format': 'uuid_hex'
                         }
                     }
-                }]
-
+                }
             }
         },
         'required': ['cpid', 'duration_seconds', 'results'],

refstack/db/migrations/alembic/utils.py

@@ -18,7 +18,13 @@ import os
 from alembic import config as alembic_conf
 from alembic.operations import Operations
 import alembic.migration as alembic_migration
-from collections import Iterable
+try:
+    # Python 3.10 and above
+    from collections.abc import Iterable
+except ImportError:
+    # Python <= 3.9, this is deprecated since Python 3.3 and it's
+    # removed in Python 3.10
+    from collections import Iterable
 from oslo_config import cfg
 from sqlalchemy import text
 

requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy>=0.8.3
-alembic
+alembic>=1.8.0
 beaker
 beautifulsoup4
 cryptography>=3.0 # BSD/Apache-2.0
@@ -11,9 +11,7 @@ oslo.utils>=3.16.0 # Apache-2.0
 pecan>=0.8.2
 requests>=2.2.0,!=2.4.0
 requests-cache>=0.4.9,<0.6.0
-# jsonschema is capped due to the following bug:
-# https://storyboard.openstack.org/#!/story/2009327
-jsonschema==3.2.0
+jsonschema>=4.7.0
 PyJWT>=2.0.0  # MIT
 WebOb>=1.7.1  # MIT
 PyMySQL>=0.6.2,!=0.6.4

setup-mysql-tests.sh

@@ -28,15 +28,21 @@ fi
 # Else setup mysql base for tests.
 # Start MySQL process for tests
 MYSQL_DATA=`mktemp -d /tmp/refstack-mysql-XXXXX`
+ls -lshd ${MYSQL_DATA}
 mkfifo ${MYSQL_DATA}/out
 # On systems like Fedora here's where mysqld can be found
 PATH=$PATH:/usr/libexec
-mysqld --no-defaults --datadir=${MYSQL_DATA} --pid-file=${MYSQL_DATA}/mysql.pid --socket=${MYSQL_DATA}/mysql.socket --skip-networking --skip-grant-tables &> ${MYSQL_DATA}/out &
+MYSQL_SOCKET="/var/run/mysqld/mysqld.sock"
+sudo chown -R mysql:mysql ${MYSQL_DATA}
+mysqld --initialize-insecure --basedir=${MYSQL_DATA} --datadir=${MYSQL_DATA}/data --pid-file=${MYSQL_DATA}/mysql.pid --socket=${MYSQL_SOCKET}/ --skip-networking --skip-grant-tables &> ${MYSQL_DATA}/out &
 # Wait for MySQL to start listening to connections
 wait_for_line "mysqld: ready for connections." ${MYSQL_DATA}/out
-export REFSTACK_TEST_MYSQL_URL="mysql+pymysql://root@localhost/test?unix_socket=${MYSQL_DATA}/mysql.socket&charset=utf8"
-mysql --no-defaults -S ${MYSQL_DATA}/mysql.socket -e 'set @@global.show_compatibility_56=ON;' > /dev/null 2>&1
-mysql --no-defaults -S ${MYSQL_DATA}/mysql.socket -e 'CREATE DATABASE test;'
+sudo mysql -S ${MYSQL_SOCKET} -e 'set @@global.show_compatibility_56=ON;' > /dev/null 2>&1
+sudo mysql -S ${MYSQL_SOCKET} -e 'CREATE DATABASE test;'
+sudo mysql -S ${MYSQL_SOCKET} -e "CREATE USER 'refstack'@'localhost' IDENTIFIED BY 'ref_pass';"
+sudo mysql -S ${MYSQL_SOCKET} -e "GRANT ALL PRIVILEGES ON test . * TO 'refstack'@'localhost';"
+sudo mysql -S ${MYSQL_SOCKET} -e "FLUSH PRIVILEGES;"
+export REFSTACK_TEST_MYSQL_URL="mysql+pymysql://refstack:ref_pass@localhost/test?unix_socket=${MYSQL_SOCKET}&charset=utf8"
 
 # Yield execution to venv command
 $*

setup.cfg

@@ -6,7 +6,7 @@ description-file =
 author = OpenStack
 author-email = openstack-discuss@lists.openstack.org
 home-page = https://refstack.openstack.org
-python_requires = >=3.6
+python_requires = >=3.8
 classifier =
     Environment :: OpenStack
     Intended Audience :: Developers
@@ -15,8 +15,6 @@ classifier =
     Operating System :: POSIX :: Linux
     Programming Language :: Python
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.6
-    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: Implementation :: CPython

test-requirements.txt

@@ -10,3 +10,4 @@ stestr>=1.1.0 # Apache-2.0
 testtools>=0.9.34
 pep257>=0.5.0
 PyMySQL>=0.6.2,!=0.6.4
+WebTest>=3.0.0

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py36,py38,pep8,pip-check-reqs
+envlist = py3,pep8,pip-check-reqs
 minversion = 3.18
 skipsdist = True