Make Tox compatible to Debian Python3.9 Environment

A different version of astroid is required for python3.9, pylint,
and pep8. The pylint setup related to pip and upper constraints
is different for python3.9. Updated the requirement packages'
software version(to versions that support python3.9) Updated
'cover' tox.ini target to be python3.9

This flake8 suppresion is added:
  H126: flag use of third party mock

These pylint disable/enable alarm checks are removed due to deprecation:
-Enable
  E1606: execfile-builtin
  E1607: old-ne-operator
  E1608: old-octal-literal
  E1610: non-ascii-bytes-literal
  E1611: invalid-unicode-literal
  W1641: eq-without-hash

-Disable
  W0403: relative-import

These pylint suppressions need to be added
  W0707: raise-missing-from
  W1406: redundant-u-string-prefix
  W1514: unspecified-encoding
  E1101: no-member

The explicit addition of python3 checks that were required
when running pylint in python2 to test python3 are removed
since pylint is now running in python3 and gets those
checks implicitly.

Removed config keys from the pylint.rc that are no longer
valid for the version of pylint executed by python3.9.

The zuul target explicitly specifies python3.9 and the
ubuntu-focal node to ensure it has the correct interpreter.

Depends-On: https://review.opendev.org/c/starlingx/distcloud/+/850543
Story: 2008943
Task: 45692

Signed-off-by: BoYuan Chang <boyuan.chang@windriver.com>
Change-Id: I70280a5fee178c228e138d4ac1c047dae7696c85

.zuul.yaml

@@ -12,6 +12,8 @@
         - stx-distcloud-tox-py27
         - stx-distcloud-tox-py39
         - stx-distcloud-tox-pylint
+        - stx-distcloud-tox-pep8-debian
+        - stx-distcloud-tox-pylint-debian
     gate:
       jobs:
         - openstack-tox-linters
@@ -19,6 +21,8 @@
         - stx-distcloud-tox-py27
         - stx-distcloud-tox-py39
         - stx-distcloud-tox-pylint
+        - stx-distcloud-tox-pep8-debian
+        - stx-distcloud-tox-pylint-debian
     post:
       jobs:
         - stx-distcloud-upload-git-mirror
@@ -69,6 +73,21 @@
       tox_envlist: pylint
       tox_extra_args: -c distributedcloud/tox.ini
 
+- job:
+    name: stx-distcloud-tox-pylint-debian
+    parent: tox
+    description: Run pylint for distcloud
+    nodeset: ubuntu-focal
+    required-projects:
+      - starlingx/fault
+      - starlingx/nfv
+      - starlingx/update
+      - starlingx/config
+    vars:
+      python_version: 3.9
+      tox_envlist: pylint_Debian
+      tox_extra_args: -c distributedcloud/tox.ini
+
 - job:
     name: stx-distcloud-tox-pep8
     parent: tox
@@ -82,6 +101,20 @@
       tox_envlist: pep8
       tox_extra_args: -c distributedcloud/tox.ini
 
+- job:
+    name: stx-distcloud-tox-pep8-debian
+    parent: tox
+    description: Run pep8 for distcloud
+    required-projects:
+      - starlingx/fault
+      - starlingx/nfv
+      - starlingx/update
+      - starlingx/config
+    vars:
+      python_version: 3.9
+      tox_envlist: pep8_Debian
+      tox_extra_args: -c distributedcloud/tox.ini
+
 - job:
     name: stx-distcloud-upload-git-mirror
     parent: upload-git-mirror

distributedcloud/pylint.rc

@@ -33,12 +33,7 @@ load-plugins=
 # E1603: unpacking-in-except
 # E1604: old-raise-syntax
 # E1605: backtick
-# E1606: long-suffix
-# E1607: old-ne-operator
-# E1608: old-octal-literal
 # E1609: import-star-module-level
-# E1610: non-ascii-bytes-literal
-# E1611: invalid-unicode-literal
 # W1601: apply-builtin
 # W1602: basestring-builtin
 # W1603: buffer-builtin
@@ -78,7 +73,6 @@ load-plugins=
 # W1638: range-builtin-not-iterating
 # W1639: filter-builtin-not-iterating
 # W1640: using-cmp-argument
-# W1641: eq-without-hash
 # W1642: div-method
 # W1643: idiv-method
 # W1644: rdiv-method
@@ -100,12 +94,12 @@ load-plugins=
 # W1660: deprecated-sys-function
 # W1661: exception-escape
 # W1662: comprehension-escape
-enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
+enable=E1603,E1609,E1602,E1605,E1604,E1601,W1652,W1651,W1649,W1657,W1660,W1658,
-       W1651,W1649,W1657,W1660,W1658,W1659,W1623,W1622,W1620,W1621,W1645,W1641,
+       W1659,W1623,W1622,W1620,W1621,W1645,W1624,W1648,W1625,W1611,W1662,W1661,
-       W1624,W1648,W1625,W1611,W1662,W1661,W1650,W1640,W1630,W1614,W1615,W1642,
+       W1650,W1640,W1630,W1614,W1615,W1642,W1616,W1628,W1643,W1629,W1627,W1644,
-       W1616,W1628,W1643,W1629,W1627,W1644,W1617,W1601,W1602,W1603,W1604,W1605,
+       W1617,W1601,W1602,W1603,W1604,W1605,W1654,W1655,W1656,W1619,W1606,W1607,
-       W1654,W1655,W1656,W1619,W1606,W1607,W1639,W1618,W1632,W1634,W1608,W1636,
+       W1639,W1618,W1632,W1634,W1608,W1636,W1653,W1646,W1638,W1609,W1610,W1626,
-       W1653,W1646,W1638,W1609,W1610,W1626,W1633,W1647,W1635,W1612,W1613,W1637
+       W1633,W1647,W1635,W1612,W1613,W1637
 
 # Disable the message, report, category or checker with the given id(s). You
 # can either give multiple identifier separated by comma (,) or put this option
@@ -128,7 +122,6 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
 # W0235: useless-super-delegation
 # W0311: bad-indentation
 # W0402: deprecated-module
-# W0403: relative-import
 # W0603: global-statement
 # W0612: unused-variable
 # W0613: unused-argument
@@ -137,14 +130,18 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
 # W0631: undefined-loop-variable
 # W0703: broad-except
 # W0706: try-except-raise
+# W0707: raise-missing-from
 # W1113: keyword-arg-before-vararg
 # W1201: logging-not-lazy
 # W1401: anomalous-backslash-in-string
+# W1406: redundant-u-string-prefix
+# W1514: unspecified-encoding
 # W1618: no-absolute-import
 disable=C,R,fixme,
-        W0102,W0105,W0107,W0123,W0201,W0211,W0212,W0221,W0223,W0231,W0235,
+        W0102,W0105,W0107,W0123,W0201,W0211,W0212,W0221,
-        W0311,W0402,W0403,W0603,W0612,W0613,W0621,W0622,W0631,W0703,W0706,
+        W0223,W0231,W0235,W0311,W0402,W0603,W0612,W0613,
-        W1113,W1201,W1401,W1618
+        W0621,W0622,W0631,W0703,W0706,W0707,W1113,W1201,
+        W1401,W1406,W1514,W1618
 
 
 [REPORTS]
@@ -152,11 +149,6 @@ disable=C,R,fixme,
 # (visual studio) and html
 output-format=text
 
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
 # Tells whether to display a full report or only the messages
 reports=no
 
@@ -212,9 +204,6 @@ generated-members=REQUEST,acl_users,aq_parent
 
 
 [BASIC]
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
 # Regular expression which should only match correct module names
 module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 
@@ -304,7 +293,7 @@ max-locals=15
 max-returns=6
 
 # Maximum number of branch for function / method body
-max-branchs=12
+max-branches=12
 
 # Maximum number of statements in function / method body
 max-statements=50

distributedcloud/test-requirements-debian.txt

@@ -0,0 +1,22 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+hacking>=1.1.0,<=2.0.0 # Apache-2.0
+astroid== 2.11.5;python_version>"3.7" # GPLv2
+cliff>=2.6.0 # Apache-2.0
+coverage!=4.4,>=4.0 # Apache-2.0
+fixtures>=3.0.0 # Apache-2.0/BSD
+mock>=2.0 # BSD
+python-subunit>=0.0.18 # Apache-2.0/BSD
+requests-mock>=1.1 # Apache-2.0
+sphinx # BSD
+oslosphinx>=4.7.0 # Apache-2.0
+testrepository>=0.0.18 # Apache-2.0/BSD
+testtools>=1.4.0 # MIT
+testresources>=0.2.4 # Apache-2.0/BSD
+testscenarios>=0.4 # Apache-2.0/BSD
+WebTest>=2.0 # MIT
+oslotest>=1.10.0 # Apache-2.0
+pylint==2.14.1;python_version>"3.7" # GPLv2
+PyYAML>=3.1.0
+yamllint<1.26.1;python_version>="3.0" # GPLv2

distributedcloud/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = pep8,py27,py39,pylint
+envlist = pep8,py27,py39,pylint,pep8_Debian,pylint_Debian
 minversion = 2.3
 skipsdist = True
 
@@ -71,7 +71,7 @@ basepython = python3.9
 install_command = pip install -v -v -v \
     -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
     {opts} {packages}
-deps = -r{toxinidir}/test-requirements.txt
+deps = -r{toxinidir}/test-requirements-debian.txt
        -r{toxinidir}/requirements.txt
        keyring
        -e{[dc]nfv_client_src_dir}
@@ -83,19 +83,22 @@ commands =
     find {toxinidir} -not -path '{toxinidir}/.tox/*' -name '*.py[c|o]' -delete
     python setup.py testr --slowest --testr-args='{posargs}'
 
-
 [testenv:debug-py27]
 basepython = python2.7
 commands = oslo_debug_helper {posargs}
 
-[testenv:debug-py34]
-basepython = python3.4
-commands = oslo_debug_helper {posargs}
-
 [testenv:pep8]
 basepython = python3
 commands = flake8
 
+[testenv:pep8_Debian]
+basepython = python3.9
+deps = -r{toxinidir}/test-requirements-debian.txt
+install_command = pip install -v -v -v \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
+    {opts} {packages}
+commands = flake8
+
 [testenv:venv]
 basepython = python3
 commands = {posargs}
@@ -113,6 +116,29 @@ commands =
     coverage xml --rcfile=.coveragerc_xml
     coverage report
 
+[testenv:cover_Debian]
+basepython = python3.9
+install_command = pip install -v -v -v \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
+    {opts} {packages}
+deps = -r{toxinidir}/test-requirements-debian.txt
+       -r{toxinidir}/requirements.txt
+       keyring
+       -e{[dc]nfv_client_src_dir}
+       -e{[dc]tsconfig_src_dir}
+       -e{[dc]fmclient_src_dir}
+       -e{[dc]fm_api_src_dir}
+       -e{[dc]cgtsclient_src_dir}
+commands =
+    coverage erase
+    rm -Rf cover
+    rm -f coverage.xml
+    find {toxinidir} -not -path '{toxinidir}/.tox/*' -name '*.py[c|o]' -delete
+    python setup.py testr --coverage --testr-args='{posargs}'
+    coverage html -d cover
+    coverage xml --rcfile=.coveragerc_xml
+    coverage report
+
 [testenv:docs]
 basepython = python3
 install_command = pip install -U {opts} {packages}
@@ -164,7 +190,8 @@ show-source = True
 # E402 module level import not at top of file
 # E501 line too long
 # E731 do not assign a lambda expression, use a def
-ignore = W503,W504,W605,
+# H216 flag use of third party mock
+ignore = W503,W504,W605,H216,
          E117,E123,E125,E305,E402,E501,E731
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build
@@ -183,6 +210,27 @@ commands =
     {[testenv]install_runtime}
     pylint {posargs} dccommon dcmanager dcorch dcdbsync --rcfile=./pylint.rc
 
+[testenv:pylint_Debian]
+basepython = python3.9
+deps = -r{toxinidir}/test-requirements-debian.txt
+       -r{toxinidir}/requirements.txt
+       -e{[dc]nfv_client_src_dir}
+       -e{[dc]tsconfig_src_dir}
+       -e{[dc]fmclient_src_dir}
+       -e{[dc]fm_api_src_dir}
+       -e{[dc]cgtsclient_src_dir}
+       -e{[dc]cgcs_patch_src_dir}
+
+install_command = pip install -v -v -v \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
+    {opts} {packages}
+
+setenv =
+  {[testenv]setenv}
+  PYTHONPATH = {toxinidir}
+commands =
+    pylint {posargs} dccommon dcmanager dcorch dcdbsync --rcfile=./pylint.rc
+
 [testenv:linters]
 basepython = python3
 # bashate ignore: