From 35b3a1da4f0c1d59718517f787f40d19111c4c3e Mon Sep 17 00:00:00 2001 From: Martin Kopec Date: Mon, 14 Dec 2020 20:48:48 +0000 Subject: [PATCH] Fix CI, Drop python2 and 3.5, update jobs to python3 * Starting in mysql8, we can't expect a db user to be created with the GRANT command, we'll need to create one explicitly. * OpenStack has dropped the py2.7 and py3.5 support in ussuri cycle. From now on, refstack will test only with py3 jobs. Complete discussion & schedule can be found in - http://lists.openstack.org/pipermail/openstack-discuss/2019-October/010142.html - https://etherpad.openstack.org/p/drop-python2-support Ussuri Communtiy-wide goal: https://governance.openstack.org/tc/goals/selected/ussuri/drop-py27.html * Replace the "refstack-tox-py36-func-mysql" with "refstack-tox-functional" to simplify naming and use the system default python3 version for this functional job. * replace testr unit test runner by stestr. testr runner hasn't been updated for ~ 6 years - stestr is its successor Story: 2003257 Task: 24062 Change-Id: I90ab2eca6366f0b6256f097712efac2ffe08b2c0 --- .gitignore | 2 +- .stestr.conf | 3 +++ .testr.conf | 4 ---- .zuul.yaml | 37 +++++++++---------------------------- requirements.txt | 2 +- setup.cfg | 4 +--- test-requirements.txt | 2 +- tools/cover.sh | 14 ++++++++++++-- tools/test-setup.sh | 4 ++-- tox.ini | 40 ++++++++++------------------------------ 10 files changed, 40 insertions(+), 72 deletions(-) create mode 100644 .stestr.conf delete mode 100644 .testr.conf diff --git a/.gitignore b/.gitignore index 13820421..b541f1d5 100755 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ *.egg* *.py[cod] .coverage -.testrepository/ +.stestr .tox/ .venv/ AUTHORS diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 00000000..672b5eba --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=./refstack/tests/unit +top_dir=./ diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index ec31a08b..00000000 --- a/.testr.conf +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./refstack -s ${SUBUNIT_TEST_PATH:-./refstack/tests/unit} $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/.zuul.yaml b/.zuul.yaml index 6afae0f8..c5af9211 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -3,44 +3,25 @@ - nodejs6-jobs - publish-openstack-docs-pti - openstack-cover-jobs - # Disable since it fails - # - openstack-python-jobs - # - openstack-python35-jobs - - openstack-python36-jobs + - openstack-python3-wallaby-jobs check: jobs: - - refstack-tox-py27-func-mysql: - voting: false - - refstack-tox-py36-func-mysql + - refstack-tox-functional: + nodeset: openstack-single-node-bionic gate: jobs: - # - refstack-tox-py27-func-mysql - - refstack-tox-py36-func-mysql + - refstack-tox-functional: + nodeset: openstack-single-node-bionic - job: - name: refstack-tox-py27-func-mysql + name: refstack-tox-functional parent: openstack-tox description: | - Run unit tests for an OpenStack Python project under cPython version 2.7. - - Uses tox with the ``py27-func-mysql`` environment. + Run functional tests for an OpenStack Python project under cPython 3. + Uses tox with the ``functional`` environment. irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - ^releasenotes/.*$ vars: - tox_envlist: py27-func-mysql - -- job: - name: refstack-tox-py36-func-mysql - parent: openstack-tox - description: | - Run unit tests for an OpenStack Python project under cPython version 3.6. - - Uses tox with the ``py36-func-mysql`` environment. - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^releasenotes/.*$ - vars: - tox_envlist: py36-func-mysql + tox_envlist: functional diff --git a/requirements.txt b/requirements.txt index 3d949eb3..c9de66e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ six>=1.9.0 # MIT pecan>=0.8.2 requests>=2.2.0,!=2.4.0 requests-cache>=0.4.9 -jsonschema>=2.0.0,<3.0.0 +jsonschema>=3.2.0 PyJWT>=1.0.1 # MIT WebOb>=1.7.1 # MIT PyMySQL>=0.6.2,!=0.6.4 diff --git a/setup.cfg b/setup.cfg index 26a80663..95c9951a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,10 +13,8 @@ classifier = License :: OSI Approved :: Apache Software License Operating System :: POSIX :: Linux Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 [files] packages = diff --git a/test-requirements.txt b/test-requirements.txt index 447ef04a..f7f07a23 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,7 +7,7 @@ httmock>=1.2.4 mock oslotest>=1.2.0 # Apache-2.0 python-subunit>=0.0.18 -testrepository>=0.0.18 +stestr>=1.1.0 # Apache-2.0 testtools>=0.9.34 pep257>=0.5.0 PyMySQL>=0.6.2,!=0.6.4 diff --git a/tools/cover.sh b/tools/cover.sh index fec62152..6ef501a1 100755 --- a/tools/cover.sh +++ b/tools/cover.sh @@ -28,7 +28,12 @@ uncommited=$(git status --porcelain | grep -v "^??") git checkout HEAD^ baseline_report=$(mktemp -t refstack_coverageXXXXXXX) -find . -type f -name "*.pyc" -delete && python setup.py testr --coverage --testr-args="$*" +coverage erase +find . -type f -name "*.pyc" -delete +stestr --test-path ./refstack/tests/unit run +coverage combine +coverage html -d cover +coverage xml -o cover/coverage.xml coverage report > $baseline_report baseline_missing=$(awk 'END { print $3 }' $baseline_report) @@ -38,7 +43,12 @@ git checkout - # Generate and save coverage report current_report=$(mktemp -t refstack_coverageXXXXXXX) -find . -type f -name "*.pyc" -delete && python setup.py testr --coverage --testr-args="$*" +coverage erase +find . -type f -name "*.pyc" -delete +stestr --test-path ./refstack/tests/unit run +coverage combine +coverage html -d cover +coverage xml -o cover/coverage.xml coverage report > $current_report current_missing=$(awk 'END { print $3 }' $current_report) diff --git a/tools/test-setup.sh b/tools/test-setup.sh index 07a07854..505a58cb 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -23,8 +23,8 @@ sudo -H mysqladmin -u root password $DB_ROOT_PW sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " DELETE FROM mysql.user WHERE User=''; FLUSH PRIVILEGES; - GRANT ALL PRIVILEGES ON *.* - TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;" + CREATE USER '$DB_USER'@'%' IDENTIFIED BY '$DB_PW'; + GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'%' WITH GRANT OPTION;" # Now create our database. mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " diff --git a/tox.ini b/tox.ini index cc1143c3..20f8e097 100644 --- a/tox.ini +++ b/tox.ini @@ -1,42 +1,32 @@ [tox] -# py3* tests should be run before py27 -# it is a workaround for testr bug -# https://bugs.launchpad.net/testrepository/+bug/1229445 -envlist = py35,py36,py27,pep8,pip-check-reqs -minversion = 1.6 +envlist = py36,pep8,pip-check-reqs +minversion = 2.0 skipsdist = True [testenv] +basepython = python3 usedevelop = True install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=C +whitelist_externals = find deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = /bin/rm -f .testrepository/times.dbm - python setup.py testr --slowest --testr-args='{posargs}' +commands = + find . -type f -name "*.pyc" -delete + stestr run {posargs} distribute = false -[testenv:py27-func-mysql] -basepython = python2.7 +[testenv:functional] +basepython = python3 setenv = SUBUNIT_TEST_PATH=./refstack/tests/api # Integration/functional tests # must not be run in parallel (--concurrency=1), # because each of these tests # require cleanup of database -commands = {toxinidir}/setup-mysql-tests.sh python setup.py testr --slowest --testr-args='{posargs:--concurrency=1}' - -[testenv:py36-func-mysql] -basepython = python3.6 -setenv = SUBUNIT_TEST_PATH=./refstack/tests/api -# Integration/functional tests -# must not be run in parallel (--concurrency=1), -# because each of these tests -# require cleanup of database -commands = {toxinidir}/setup-mysql-tests.sh python setup.py testr --slowest --testr-args='{posargs:--concurrency=1}' - +commands = {toxinidir}/setup-mysql-tests.sh stestr run --test-path ./refstack/tests/api {posargs:--concurrency=1} [testenv:pep8] commands = @@ -55,11 +45,6 @@ commands = [testenv:venv] commands = {posargs} -[testenv:gen-cover] -commands = python setup.py testr --coverage \ - --omit='{toxinidir}/refstack/tests/unit/*,{toxinidir}/refstack/tests/api/*,{toxinidir}/refstack/api/config.py,{toxinidir}/refstack/db/migrations/alembic/*,{toxinidir}/refstack/opts.py' \ - --testr-args='{posargs}' - [testenv:cover] commands = {toxinidir}/tools/cover.sh {posargs} @@ -90,12 +75,7 @@ commands= [testenv:debug] commands = oslo_debug_helper -t refstack/tests/unit {posargs} -[testenv:debug35] -basepython = python3.5 -commands = oslo_debug_helper -t refstack/tests/unit {posargs} - [testenv:bindep] -basepython = python3 # Do not install any requirements. We want this to be fast and work even if # system dependencies are missing, since it's used to tell you what system # dependencies are missing! This also means that bindep must be installed