From eafee58e62b273aa3b65e497a4e4efc89f9e0715 Mon Sep 17 00:00:00 2001 From: albailey Date: Tue, 16 Feb 2021 12:15:30 -0600 Subject: [PATCH] Not require recreate of tox env when running tox The upgrade test was written to copy a file required for the test, but did not handle the file already being processed. This fix adjusts the setup for the unit test to handle this and allows the recreate to be removed, thus reducing designer tox testing time (40 seconds down to 4 when re-running a single test) This also aligns the version of hacking and pylint as the same as other starlingx repos. A new pylint was released which was breaking tox. The code changes are to fix whitespace (rather than suppressing): E305 expected 2 blank lines after class or function definition E117 over-indented Three new hacking warnings are being suppressed: W503 line break before binary operator W504 line break after binary operator W605 invalid escape sequence Un-suppress these codes which are not broken in the code: E123 closing bracket does not match indentation of opening bracket E501 line too long (tox sets max length instead to a large val) F823 local variable referenced before assignment yamllint 1.26.1 does not work with py2 or py3 in our env, so clamp it. Story: 2008137 Task: 42200 Signed-off-by: albailey Change-Id: Ia623315c5f1a690d20f597242b7722601776f26d --- .../histogram_analysis/plotter.py | 2 +- .../tests/test_database_upgrades.py | 20 ++++++-- nfv/nfv-vim/nfv_vim/api/openstack/_objects.py | 1 + .../network_rebalance/_dhcp_rebalance.py | 2 + .../network_rebalance/_network_rebalance.py | 2 + .../nfv_vim/strategy/_strategy_defs.py | 1 + nfv/test-requirements.txt | 14 +++++- nfv/tox.ini | 46 ++++++++++--------- nova-api-proxy/test-requirements.txt | 12 +++++ nova-api-proxy/tox.ini | 10 ++-- test-requirements.txt | 6 +-- tox.ini | 2 +- 12 files changed, 81 insertions(+), 37 deletions(-) create mode 100644 nova-api-proxy/test-requirements.txt diff --git a/nfv/nfv-debug-tools/histogram_analysis/plotter.py b/nfv/nfv-debug-tools/histogram_analysis/plotter.py index dd6088af..223b94ae 100755 --- a/nfv/nfv-debug-tools/histogram_analysis/plotter.py +++ b/nfv/nfv-debug-tools/histogram_analysis/plotter.py @@ -348,7 +348,7 @@ def setFilename(graphName): os.makedirs("Graphs/") os.chdir(os.path.join(dir, 'Graphs/')) if not graphName: - graphName = time.strftime("%m-%d-%Y") + graphName = time.strftime("%m-%d-%Y") if os.path.exists(str(graphName + ".html")): n = 1 while(not validName): diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_database_upgrades.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_database_upgrades.py index 5098ec53..876c9111 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/test_database_upgrades.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_database_upgrades.py @@ -4,7 +4,9 @@ # SPDX-License-Identifier: Apache-2.0 # import os +import shutil import subprocess +import tempfile from nfv_vim import database from nfv_vim import tables @@ -14,24 +16,34 @@ from . import testcase # noqa: H304 class TestNFVDatabaseUpgrade(testcase.NFVTestCase): + def setUp(self): + super(TestNFVDatabaseUpgrade, self).setUp() + root_dir = os.environ['VIRTUAL_ENV'] + # create a directory to hold the DB, randomly named, under the tox env + self.db_dir = tempfile.mkdtemp(dir=root_dir) + + def tearDown(self): + super(TestNFVDatabaseUpgrade, self).tearDown() + shutil.rmtree(self.db_dir) + def test_nfv_vim_database_upgrade_from_19_12(self): """ Test VIM database upgrades from stx 19_12 """ root_dir = os.environ['VIRTUAL_ENV'] - + # stage some old data devnull = open(os.devnull, 'w') try: vim_cmd = ("nfv-vim-manage db-load-data -d %s " - "-f %s/nfv_vim_db_stx_19.12" % (root_dir, root_dir)) + "-f %s/nfv_vim_db_stx_19.12" % (self.db_dir, root_dir)) subprocess.check_call([vim_cmd], shell=True, stderr=devnull) - except subprocess.CalledProcessError: raise + # migrate the old data config = dict() - config['database_dir'] = root_dir + config['database_dir'] = self.db_dir database.database_initialize(config) database.database_migrate_data() tables.tables_initialize() diff --git a/nfv/nfv-vim/nfv_vim/api/openstack/_objects.py b/nfv/nfv-vim/nfv_vim/api/openstack/_objects.py index aed49c86..5e94298c 100755 --- a/nfv/nfv-vim/nfv_vim/api/openstack/_objects.py +++ b/nfv/nfv-vim/nfv_vim/api/openstack/_objects.py @@ -60,6 +60,7 @@ class OpenStackServices(Constants): HEAT = Constant('heat') FM = Constant('fm') + # OpenStack Services Constant OPENSTACK_SERVICE = OpenStackServices() diff --git a/nfv/nfv-vim/nfv_vim/network_rebalance/_dhcp_rebalance.py b/nfv/nfv-vim/nfv_vim/network_rebalance/_dhcp_rebalance.py index 6de901a8..046e969e 100644 --- a/nfv/nfv-vim/nfv_vim/network_rebalance/_dhcp_rebalance.py +++ b/nfv/nfv-vim/nfv_vim/network_rebalance/_dhcp_rebalance.py @@ -27,6 +27,7 @@ class AgentType(Constants): L3 = Constant('L3 agent') DHCP = Constant('DHCP agent') + AGENT_TYPE = AgentType() @@ -43,6 +44,7 @@ class DHCPRebalanceState(Constants): HOLD_OFF = Constant('HOLD_OFF') DONE = Constant('DONE') + DHCP_REBALANCE_STATE = DHCPRebalanceState() diff --git a/nfv/nfv-vim/nfv_vim/network_rebalance/_network_rebalance.py b/nfv/nfv-vim/nfv_vim/network_rebalance/_network_rebalance.py index 06258e7f..13506db8 100644 --- a/nfv/nfv-vim/nfv_vim/network_rebalance/_network_rebalance.py +++ b/nfv/nfv-vim/nfv_vim/network_rebalance/_network_rebalance.py @@ -27,6 +27,7 @@ class AgentType(Constants): L3 = Constant('L3 agent') DHCP = Constant('DHCP agent') + AGENT_TYPE = AgentType() @@ -46,6 +47,7 @@ class L3RebalanceState(Constants): HOLD_OFF = Constant('HOLD_OFF') DONE = Constant('DONE') + L3_REBALANCE_STATE = L3RebalanceState() diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py index ad258b1d..b137a304 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py @@ -54,4 +54,5 @@ class FirmwareUpdateLabels(object): DEVICE_IMAGE_UPDATE_FAILED = Constant('failed') DEVICE_IMAGE_UPDATE_IN_PROGRESS_ABORTED = Constant('in-progress-aborted') + FW_UPDATE_LABEL = FirmwareUpdateLabels() diff --git a/nfv/test-requirements.txt b/nfv/test-requirements.txt index c3ed418b..06cc8f76 100644 --- a/nfv/test-requirements.txt +++ b/nfv/test-requirements.txt @@ -1 +1,13 @@ -hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +hacking>=1.1.0,<=2.0.0 # Apache-2.0 +astroid<2.0;python_version<"3.0" # GPLv2 +astroid<= 2.2.5;python_version>="3.0" # GPLv2 +bandit<1.6.0;python_version>="3.0" +coverage>=3.6 +fixtures>=3.0.0 # Apache-2.0/BSD +mock>=2.0.0 # BSD +pylint<2.1.0;python_version<"3.0" # GPLv2 +pylint<2.4.0;python_version>="3.0" # GPLv2 +testtools>=2.2.0 # MIT diff --git a/nfv/tox.ini b/nfv/tox.ini index 84652c09..2812d90c 100644 --- a/nfv/tox.ini +++ b/nfv/tox.ini @@ -15,12 +15,14 @@ skipsdist = True stxdir = {toxinidir}/../.. [testenv] -recreate = True install_command = pip install \ -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \ {opts} {packages} whitelist_externals = cp +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + [nfv] nfv_base_dir = . nfv_client_dir = ./nfv-client @@ -57,30 +59,35 @@ deps = {[nfv]nfv_client_dir} verbosity=2 [flake8] -# Temporarily ignoring these warnings -# E121 continuation line under-indented for hanging indent -# E122 continuation line missing indentation or outdented -# E123 closing bracket does not match indentation of opening bracket -# E124 closing bracket does not match visual indentation -# E126 continuation line over-indented for hanging indent -# E127 continuation line over-indented for visual indent -# E128 continuation line under-indented for visual indent -# E129 visually indented line with same indent as next logical line -# E501 line too long # - hacking codes - # H104: File contains nothing but comments # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line # H501: Do not use self.__dict__ for string formatting -# - flake8 codes - -# F823 local variable referenced before assignment +# warnings +# W503 line break before binary operator +# W504 line break after binary operator +# W605 invalid escape sequence +# errors +# E121 continuation line under-indented for hanging indent +# E122 continuation line missing indentation or outdented +# E124 closing bracket does not match visual indentation +# E126 continuation line over-indented for hanging indent +# E127 continuation line over-indented for visual indent +# E128 continuation line under-indented for visual indent +# E129 visually indented line with same indent as next logical line +# flake8 # F841 local variable 'e' is assigned to but never used -ignore = E121,E122,E123,E124,E126,E127,E128,E129,E501,H104,H404,H405,H501,F823,F841 +ignore = H104,H404,H405,H501, + W503,W504,W605, + E121,E122,E124,E126,E127,E128,E129, + F841 # H106 Don’t put vim configuration in source files (off by default). # H203 Use assertIs(Not)None to check for None (off by default). # TODO: enable: H904 Delay string interpolations at logging calls (off by default). enable-extensions = H106,H203 -max-line-length = 84 +# rather than suppress E501 line too long, set max line length and reduce later +max-line-length = 185 [testenv:pep8] basepython = python3 @@ -94,12 +101,9 @@ commands = [testenv:pylint] basepython = python2.7 deps = {[nfv]deps} + -r{toxinidir}/test-requirements.txt {[nfv]nfv_test_dir}/nfv_scenario_tests - fixtures jinja2 - mock - testtools - pylint commands = pylint {[nfv]nfv_client_src_dir} \ {[nfv]nfv_common_src_dir} \ {[nfv]nfv_plugins_src_dir} \ @@ -120,8 +124,8 @@ exclude = ./nfv-docs,./nfv-tests,./nfv-debug-tools,unit_test [testenv:bandit] deps = {[nfv]deps} - bandit -basepython = python2.7 + -r{toxinidir}/test-requirements.txt +basepython = python3 commands = bandit --ini tox.ini -f txt -r {[nfv]nfv_base_dir} [testenv:py27] diff --git a/nova-api-proxy/test-requirements.txt b/nova-api-proxy/test-requirements.txt new file mode 100644 index 00000000..2e3f81d3 --- /dev/null +++ b/nova-api-proxy/test-requirements.txt @@ -0,0 +1,12 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +hacking>=1.1.0,<=2.0.0 # Apache-2.0 +astroid<2.0;python_version<"3.0" # GPLv2 +astroid<= 2.2.5;python_version>="3.0" # GPLv2 +bandit<1.6.0;python_version>="3.0" +coverage>=3.6 +fixtures>=3.0.0 # Apache-2.0/BSD +mock>=2.0.0 # BSD +pylint<2.1.0;python_version<"3.0" # GPLv2 +pylint<2.4.0;python_version>="3.0" # GPLv2 diff --git a/nova-api-proxy/tox.ini b/nova-api-proxy/tox.ini index e4073772..f06f1bc2 100644 --- a/nova-api-proxy/tox.ini +++ b/nova-api-proxy/tox.ini @@ -5,6 +5,7 @@ skipsdist = True [testenv] basepython = python3 +sitepackages = False install_command = pip install \ -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \ {opts} {packages} @@ -16,6 +17,8 @@ setenv = VIRTUAL_ENV={envdir} commands = find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -not -path '*/__pycache__/*' -name '*.py[c|o]' -delete whitelist_externals = find +deps=-r{toxinidir}/test-requirements.txt + [testenv:venv] commands = {posargs} @@ -36,15 +39,13 @@ max-line-length=84 [testenv:pep8] usedevelop = False skip_install = True -deps = - hacking - flake8-bugbear commands = flake8 [testenv:pylint] basepython = python2.7 -deps = +deps= + -r{toxinidir}/test-requirements.txt eventlet oslo.config oslo.log @@ -52,6 +53,5 @@ deps = PasteDeploy routes webob - pylint commands = pylint nova-api-proxy/nova_api_proxy --rcfile=pylint.rc diff --git a/test-requirements.txt b/test-requirements.txt index 67d9ddf1..0fbded31 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,4 @@ bashate < 1.0.0 PyYAML >= 3.1.0 -yamllint >= 0.5.2 -# The following are for py27 and py35 -mock # BSD -nose # LGPL +yamllint<1.26.1;python_version>="3.0" # GPLv2 + diff --git a/tox.ini b/tox.ini index 150ac999..e5e39b5e 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,7 @@ basepython = python3 description = Dummy environment to allow pep8 to be run in subdir tox [testenv:pylint] -basepython = python3 +basepython = python2.7 description = Dummy environment to allow pylint to be run in subdir tox [testenv:linters]