Fix CI, Drop python2 and 3.5, update jobs to python3

* Starting in mysql8, we can't expect a db user to be
  created with the GRANT command, we'll need to create
  one explicitly.

* OpenStack has dropped the py2.7 and py3.5 support
  in ussuri cycle. From now on, refstack will test only
  with py3 jobs. Complete discussion & schedule can be
  found in
  - http://lists.openstack.org/pipermail/openstack-discuss/2019-October/010142.html
  - https://etherpad.openstack.org/p/drop-python2-support

  Ussuri Communtiy-wide goal:
  https://governance.openstack.org/tc/goals/selected/ussuri/drop-py27.html

* Replace the "refstack-tox-py36-func-mysql" with
  "refstack-tox-functional" to simplify naming and use the system default
  python3 version for this functional job.

* replace testr unit test runner by stestr. testr runner hasn't been updated
  for ~ 6 years - stestr is its successor

Story: 2003257
Task: 24062
Change-Id: I90ab2eca6366f0b6256f097712efac2ffe08b2c0

.gitignore

@@ -1,7 +1,7 @@
 *.egg*
 *.py[cod]
 .coverage
-.testrepository/
+.stestr
 .tox/
 .venv/
 AUTHORS

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./refstack/tests/unit
+top_dir=./

.testr.conf

@@ -1,4 +0,0 @@
-[DEFAULT]
-test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./refstack -s ${SUBUNIT_TEST_PATH:-./refstack/tests/unit} $LISTOPT $IDOPTION
-test_id_option=--load-list $IDFILE
-test_list_option=--list

.zuul.yaml

@@ -3,44 +3,25 @@
       - nodejs6-jobs
       - publish-openstack-docs-pti
       - openstack-cover-jobs
-      # Disable since it fails
+      - openstack-python3-wallaby-jobs
-      # - openstack-python-jobs
-      # - openstack-python35-jobs
-      - openstack-python36-jobs
     check:
       jobs:
-        - refstack-tox-py27-func-mysql:
+        - refstack-tox-functional:
-            voting: false
+            nodeset: openstack-single-node-bionic
-        - refstack-tox-py36-func-mysql
     gate:
       jobs:
-        # - refstack-tox-py27-func-mysql
+        - refstack-tox-functional:
-        - refstack-tox-py36-func-mysql
+            nodeset: openstack-single-node-bionic
 
 - job:
-    name: refstack-tox-py27-func-mysql
+    name: refstack-tox-functional
     parent: openstack-tox
     description: |
-      Run unit tests for an OpenStack Python project under cPython version 2.7.
+      Run functional tests for an OpenStack Python project under cPython 3.
-
+      Uses tox with the ``functional`` environment.
-      Uses tox with the ``py27-func-mysql`` environment.
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*$
       - ^releasenotes/.*$
     vars:
-      tox_envlist: py27-func-mysql
+      tox_envlist: functional
-
-- job:
-    name: refstack-tox-py36-func-mysql
-    parent: openstack-tox
-    description: |
-      Run unit tests for an OpenStack Python project under cPython version 3.6.
-
-      Uses tox with the ``py36-func-mysql`` environment.
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^doc/.*$
-      - ^releasenotes/.*$
-    vars:
-      tox_envlist: py36-func-mysql

requirements.txt

@@ -12,7 +12,7 @@ six>=1.9.0 # MIT
 pecan>=0.8.2
 requests>=2.2.0,!=2.4.0
 requests-cache>=0.4.9
-jsonschema>=2.0.0,<3.0.0
+jsonschema>=3.2.0
 PyJWT>=1.0.1  # MIT
 WebOb>=1.7.1  # MIT
 PyMySQL>=0.6.2,!=0.6.4

setup.cfg

@@ -13,10 +13,8 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python
-    Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.6
 
 [files]
 packages =

test-requirements.txt

@@ -7,7 +7,7 @@ httmock>=1.2.4
 mock
 oslotest>=1.2.0 # Apache-2.0
 python-subunit>=0.0.18
-testrepository>=0.0.18
+stestr>=1.1.0 # Apache-2.0
 testtools>=0.9.34
 pep257>=0.5.0
 PyMySQL>=0.6.2,!=0.6.4

tools/cover.sh

@@ -28,7 +28,12 @@ uncommited=$(git status --porcelain | grep -v "^??")
 git checkout HEAD^
 
 baseline_report=$(mktemp -t refstack_coverageXXXXXXX)
-find . -type f -name "*.pyc" -delete && python setup.py testr --coverage --testr-args="$*"
+coverage erase
+find . -type f -name "*.pyc" -delete
+stestr --test-path ./refstack/tests/unit run
+coverage combine
+coverage html -d cover
+coverage xml -o cover/coverage.xml
 coverage report > $baseline_report
 baseline_missing=$(awk 'END { print $3 }' $baseline_report)
 
@@ -38,7 +43,12 @@ git checkout -
 
 # Generate and save coverage report
 current_report=$(mktemp -t refstack_coverageXXXXXXX)
-find . -type f -name "*.pyc" -delete && python setup.py testr --coverage --testr-args="$*"
+coverage erase
+find . -type f -name "*.pyc" -delete
+stestr --test-path ./refstack/tests/unit run
+coverage combine
+coverage html -d cover
+coverage xml -o cover/coverage.xml
 coverage report > $current_report
 current_missing=$(awk 'END { print $3 }' $current_report)
 

tools/test-setup.sh

@@ -23,8 +23,8 @@ sudo -H mysqladmin -u root password $DB_ROOT_PW
 sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e "
     DELETE FROM mysql.user WHERE User='';
     FLUSH PRIVILEGES;
-    GRANT ALL PRIVILEGES ON *.*
+    CREATE USER '$DB_USER'@'%' IDENTIFIED BY '$DB_PW';
-        TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;"
+    GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'%' WITH GRANT OPTION;"
 
 # Now create our database.
 mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "

tox.ini

@@ -1,42 +1,32 @@
 [tox]
-# py3* tests should be run before py27
+envlist = py36,pep8,pip-check-reqs
-# it is a workaround for testr bug
+minversion = 2.0
-# https://bugs.launchpad.net/testrepository/+bug/1229445
-envlist = py35,py36,py27,pep8,pip-check-reqs
-minversion = 1.6
 skipsdist = True
 
 [testenv]
+basepython = python3
 usedevelop = True
 install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} -U {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
          LANG=en_US.UTF-8
          LANGUAGE=en_US:en
          LC_ALL=C
+whitelist_externals = find
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-commands = /bin/rm -f .testrepository/times.dbm
+commands =
-           python setup.py testr --slowest --testr-args='{posargs}'
+    find . -type f -name "*.pyc" -delete
+    stestr run {posargs}
 distribute = false
 
-[testenv:py27-func-mysql]
+[testenv:functional]
-basepython = python2.7
+basepython = python3
 setenv = SUBUNIT_TEST_PATH=./refstack/tests/api
 # Integration/functional tests
 # must not be run in parallel (--concurrency=1),
 # because each of these tests
 # require cleanup of database
-commands = {toxinidir}/setup-mysql-tests.sh python setup.py testr --slowest --testr-args='{posargs:--concurrency=1}'
+commands = {toxinidir}/setup-mysql-tests.sh stestr run --test-path ./refstack/tests/api {posargs:--concurrency=1}
-
-[testenv:py36-func-mysql]
-basepython = python3.6
-setenv = SUBUNIT_TEST_PATH=./refstack/tests/api
-# Integration/functional tests
-# must not be run in parallel (--concurrency=1),
-# because each of these tests
-# require cleanup of database
-commands = {toxinidir}/setup-mysql-tests.sh python setup.py testr --slowest --testr-args='{posargs:--concurrency=1}'
-
 
 [testenv:pep8]
 commands =
@@ -55,11 +45,6 @@ commands =
 [testenv:venv]
 commands = {posargs}
 
-[testenv:gen-cover]
-commands = python setup.py testr --coverage \
-    --omit='{toxinidir}/refstack/tests/unit/*,{toxinidir}/refstack/tests/api/*,{toxinidir}/refstack/api/config.py,{toxinidir}/refstack/db/migrations/alembic/*,{toxinidir}/refstack/opts.py' \
-    --testr-args='{posargs}'
-
 [testenv:cover]
 commands = {toxinidir}/tools/cover.sh {posargs}
 
@@ -90,12 +75,7 @@ commands=
 [testenv:debug]
 commands = oslo_debug_helper -t refstack/tests/unit {posargs}
 
-[testenv:debug35]
-basepython = python3.5
-commands = oslo_debug_helper -t refstack/tests/unit {posargs}
-
 [testenv:bindep]
-basepython = python3
 # Do not install any requirements. We want this to be fast and work even if
 # system dependencies are missing, since it's used to tell you what system
 # dependencies are missing! This also means that bindep must be installed