Not require recreate of tox env when running tox

The upgrade test was written to copy a file required for
the test, but did not handle the file already being processed.
This fix adjusts the setup for the unit test to handle this
and allows the recreate to be removed, thus reducing designer
tox testing time (40 seconds down to 4 when re-running a single test)

This also aligns the version of hacking and pylint as the same as
other starlingx repos. A new pylint was released which was
breaking tox.

The code changes are to fix whitespace (rather than suppressing):
 E305 expected 2 blank lines after class or function definition
 E117 over-indented

Three new hacking warnings are being suppressed:
  W503 line break before binary operator
  W504  line break after binary operator
  W605 invalid escape sequence

Un-suppress these codes which are not broken in the code:
 E123 closing bracket does not match indentation of opening bracket
 E501 line too long (tox sets max length instead to a large val)
 F823 local variable referenced before assignment

yamllint 1.26.1 does not work with py2 or py3 in our env, so clamp it.

Story: 2008137
Task: 42200
Signed-off-by: albailey <Al.Bailey@windriver.com>
Change-Id: Ia623315c5f1a690d20f597242b7722601776f26d

nfv/nfv-tests/nfv_unit_tests/tests/test_database_upgrades.py

@@ -4,7 +4,9 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 import os
+import shutil
 import subprocess
+import tempfile
 
 from nfv_vim import database
 from nfv_vim import tables
@@ -14,24 +16,34 @@ from . import testcase  # noqa: H304
 
 class TestNFVDatabaseUpgrade(testcase.NFVTestCase):
 
+    def setUp(self):
+        super(TestNFVDatabaseUpgrade, self).setUp()
+        root_dir = os.environ['VIRTUAL_ENV']
+        # create a directory to hold the DB, randomly named, under the tox env
+        self.db_dir = tempfile.mkdtemp(dir=root_dir)
+
+    def tearDown(self):
+        super(TestNFVDatabaseUpgrade, self).tearDown()
+        shutil.rmtree(self.db_dir)
+
     def test_nfv_vim_database_upgrade_from_19_12(self):
         """
         Test VIM database upgrades from stx 19_12
         """
         root_dir = os.environ['VIRTUAL_ENV']
-
+        # stage some old data
         devnull = open(os.devnull, 'w')
         try:
             vim_cmd = ("nfv-vim-manage db-load-data -d %s "
-                       "-f %s/nfv_vim_db_stx_19.12" % (root_dir, root_dir))
+                       "-f %s/nfv_vim_db_stx_19.12" % (self.db_dir, root_dir))
 
             subprocess.check_call([vim_cmd], shell=True, stderr=devnull)
-
         except subprocess.CalledProcessError:
             raise
 
+        # migrate the old data
         config = dict()
-        config['database_dir'] = root_dir
+        config['database_dir'] = self.db_dir
         database.database_initialize(config)
         database.database_migrate_data()
         tables.tables_initialize()

nfv/nfv-vim/nfv_vim/api/openstack/_objects.py

@@ -60,6 +60,7 @@ class OpenStackServices(Constants):
     HEAT = Constant('heat')
     FM = Constant('fm')
 
+
 # OpenStack Services Constant
 OPENSTACK_SERVICE = OpenStackServices()
 

nfv/nfv-vim/nfv_vim/network_rebalance/_dhcp_rebalance.py

@@ -27,6 +27,7 @@ class AgentType(Constants):
     L3 = Constant('L3 agent')
     DHCP = Constant('DHCP agent')
 
+
 AGENT_TYPE = AgentType()
 
 
@@ -43,6 +44,7 @@ class DHCPRebalanceState(Constants):
     HOLD_OFF = Constant('HOLD_OFF')
     DONE = Constant('DONE')
 
+
 DHCP_REBALANCE_STATE = DHCPRebalanceState()
 
 

nfv/nfv-vim/nfv_vim/network_rebalance/_network_rebalance.py

@@ -27,6 +27,7 @@ class AgentType(Constants):
     L3 = Constant('L3 agent')
     DHCP = Constant('DHCP agent')
 
+
 AGENT_TYPE = AgentType()
 
 
@@ -46,6 +47,7 @@ class L3RebalanceState(Constants):
     HOLD_OFF = Constant('HOLD_OFF')
     DONE = Constant('DONE')
 
+
 L3_REBALANCE_STATE = L3RebalanceState()
 
 

nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py

@@ -54,4 +54,5 @@ class FirmwareUpdateLabels(object):
     DEVICE_IMAGE_UPDATE_FAILED = Constant('failed')
     DEVICE_IMAGE_UPDATE_IN_PROGRESS_ABORTED = Constant('in-progress-aborted')
 
+
 FW_UPDATE_LABEL = FirmwareUpdateLabels()

nfv/test-requirements.txt

@@ -1 +1,13 @@
-hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+hacking>=1.1.0,<=2.0.0 # Apache-2.0
+astroid<2.0;python_version<"3.0" # GPLv2
+astroid<= 2.2.5;python_version>="3.0" # GPLv2
+bandit<1.6.0;python_version>="3.0"
+coverage>=3.6
+fixtures>=3.0.0 # Apache-2.0/BSD
+mock>=2.0.0 # BSD
+pylint<2.1.0;python_version<"3.0" # GPLv2
+pylint<2.4.0;python_version>="3.0" # GPLv2
+testtools>=2.2.0 # MIT

nfv/tox.ini

@@ -15,12 +15,14 @@ skipsdist = True
 stxdir = {toxinidir}/../..
 
 [testenv]
-recreate = True
 install_command = pip install \
     -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \
     {opts} {packages}
 whitelist_externals = cp
 
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+
 [nfv]
 nfv_base_dir = .
 nfv_client_dir = ./nfv-client
@@ -57,30 +59,35 @@ deps = {[nfv]nfv_client_dir}
 verbosity=2
 
 [flake8]
-# Temporarily ignoring these warnings
-# E121 continuation line under-indented for hanging indent
-# E122 continuation line missing indentation or outdented
-# E123 closing bracket does not match indentation of opening bracket
-# E124 closing bracket does not match visual indentation
-# E126 continuation line over-indented for hanging indent
-# E127 continuation line over-indented for visual indent
-# E128 continuation line under-indented for visual indent
-# E129 visually indented line with same indent as next logical line
-# E501 line too long
 # - hacking codes -
 # H104: File contains nothing but comments
 # H404: multi line docstring should start without a leading new line
 # H405: multi line docstring summary not separated with an empty line
 # H501: Do not use self.__dict__ for string formatting
-# - flake8 codes -
-# F823 local variable referenced before assignment
+# warnings
+# W503 line break before binary operator
+# W504  line break after binary operator
+# W605 invalid escape sequence
+# errors
+# E121 continuation line under-indented for hanging indent
+# E122 continuation line missing indentation or outdented
+# E124 closing bracket does not match visual indentation
+# E126 continuation line over-indented for hanging indent
+# E127 continuation line over-indented for visual indent
+# E128 continuation line under-indented for visual indent
+# E129 visually indented line with same indent as next logical line
+# flake8
 # F841 local variable 'e' is assigned to but never used
-ignore = E121,E122,E123,E124,E126,E127,E128,E129,E501,H104,H404,H405,H501,F823,F841
+ignore = H104,H404,H405,H501,
+         W503,W504,W605,
+         E121,E122,E124,E126,E127,E128,E129,
+         F841
 # H106 Don’t put vim configuration in source files (off by default).
 # H203 Use assertIs(Not)None to check for None (off by default).
 # TODO: enable: H904 Delay string interpolations at logging calls (off by default).
 enable-extensions = H106,H203
-max-line-length = 84
+# rather than suppress E501 line too long, set max line length and reduce later
+max-line-length = 185
 
 [testenv:pep8]
 basepython = python3
@@ -94,12 +101,9 @@ commands =
 [testenv:pylint]
 basepython = python2.7
 deps = {[nfv]deps}
+       -r{toxinidir}/test-requirements.txt
        {[nfv]nfv_test_dir}/nfv_scenario_tests
-       fixtures
        jinja2
-       mock
-       testtools
-       pylint
 commands = pylint {[nfv]nfv_client_src_dir}  \
                   {[nfv]nfv_common_src_dir}  \
                   {[nfv]nfv_plugins_src_dir} \
@@ -120,8 +124,8 @@ exclude = ./nfv-docs,./nfv-tests,./nfv-debug-tools,unit_test
 
 [testenv:bandit]
 deps = {[nfv]deps}
-       bandit
-basepython = python2.7
+       -r{toxinidir}/test-requirements.txt
+basepython = python3
 commands = bandit --ini tox.ini -f txt -r {[nfv]nfv_base_dir}
 
 [testenv:py27]

nova-api-proxy/test-requirements.txt

@@ -0,0 +1,12 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+hacking>=1.1.0,<=2.0.0 # Apache-2.0
+astroid<2.0;python_version<"3.0" # GPLv2
+astroid<= 2.2.5;python_version>="3.0" # GPLv2
+bandit<1.6.0;python_version>="3.0"
+coverage>=3.6
+fixtures>=3.0.0 # Apache-2.0/BSD
+mock>=2.0.0 # BSD
+pylint<2.1.0;python_version<"3.0" # GPLv2
+pylint<2.4.0;python_version>="3.0" # GPLv2

nova-api-proxy/tox.ini

@@ -5,6 +5,7 @@ skipsdist = True
 
 [testenv]
 basepython = python3
+sitepackages = False
 install_command = pip install \
     -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \
     {opts} {packages}
@@ -16,6 +17,8 @@ setenv = VIRTUAL_ENV={envdir}
 commands = find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -not -path '*/__pycache__/*' -name '*.py[c|o]' -delete
 whitelist_externals = find
 
+deps=-r{toxinidir}/test-requirements.txt
+
 [testenv:venv]
 commands = {posargs}
 
@@ -36,15 +39,13 @@ max-line-length=84
 [testenv:pep8]
 usedevelop = False
 skip_install = True
-deps =
-    hacking
-    flake8-bugbear
 commands =
     flake8
 
 [testenv:pylint]
 basepython = python2.7
-deps =
+deps=
+    -r{toxinidir}/test-requirements.txt
     eventlet
     oslo.config
     oslo.log
@@ -52,6 +53,5 @@ deps =
     PasteDeploy
     routes
     webob
-    pylint
 commands = pylint nova-api-proxy/nova_api_proxy --rcfile=pylint.rc
 

test-requirements.txt

@@ -1,6 +1,4 @@
 bashate < 1.0.0
 PyYAML >= 3.1.0
-yamllint >= 0.5.2
-# The following are for py27 and py35
-mock  # BSD
-nose  # LGPL
+yamllint<1.26.1;python_version>="3.0" # GPLv2
+

tox.ini

@@ -27,7 +27,7 @@ basepython = python3
 description = Dummy environment to allow pep8 to be run in subdir tox
 
 [testenv:pylint]
-basepython = python3
+basepython = python2.7
 description = Dummy environment to allow pylint to be run in subdir tox
 
 [testenv:linters]