Update sysinv tox for python3.9

sysinv will now use python3.9 for pylint, flake8, bandit basepython is now set to python3.9 The zuul jobs for older versions are now updated or removed The nodeset for python 3.9 jobs is set to debian-bullseye pylint.rc changes include: - Upversion to latest pylint (2.16) - remove suppressions that no longer exist. - add suppressions for new checks added in recent pylint - update pylint.rc settings that no longer exist. - remove explicit enabling of pylint checkers from python2 that are on by default in python3 Note: some of the newly suppressed pylint error codes will be un-suppressed in later commits. A subtle issue has also been fixed where a variable was shadowing an imported method, and would have led to a an exception when logging an error. The variable has been renamed. Example CLI service-parameter command for that code path: system service-parameter-modify kubernetes \ kube_apiserver_volumes \ default-audit-policy-file='{"hostPath":"x", "name":"y"}' The CLI of the previous code would show the following in its exception case without the fix 'int' object is not callable Test Plan: PASS: run tox in a debian build environment. PASS: build/install/unlock AIO-SX PASS: service-parameter CLI command still works Story: 2010531 Task: 47279 Change-Id: I989ec6e75463be0a4dcd2c7727cd69584c2a20a2 Signed-off-by: Al Bailey <al.bailey@windriver.com>
2022-07-21 15:11:52 +00:00 · 2022-07-21 15:11:52 +00:00 · 4927790c0d
commit 4927790c0d
parent e9705f5bc6
5 changed files with 49 additions and 164 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -9,7 +9,6 @@
    check:
      jobs:
        - openstack-tox-linters
-        - sysinv-tox-py27
        - sysinv-tox-py39
        - sysinv-tox-flake8
        - sysinv-tox-pylint
@ -29,7 +28,6 @@
    gate:
      jobs:
        - openstack-tox-linters
-        - sysinv-tox-py27
        - sysinv-tox-py39
        - sysinv-tox-flake8
        - sysinv-tox-pylint
@ -50,22 +48,6 @@
      jobs:
        - stx-config-upload-git-mirror

- job:
-    name: sysinv-tox-py27
-    parent: tox
-    description: |
-      Run py27 test for sysinv
-    nodeset: ubuntu-xenial
-    required-projects:
-      - starlingx/fault
-      - starlingx/utilities
-      - starlingx/update
-    files:
-      - sysinv/sysinv/*
-    vars:
-      tox_envlist: py27
-      tox_extra_args: -c sysinv/sysinv/sysinv/tox.ini
-
 - job:
    name: sysinv-tox-py39
    parent: tox-py39
@ -88,7 +70,7 @@
    parent: tox
    description: |
      Run flake8 test for sysinv
-    nodeset: ubuntu-bionic
+    nodeset: debian-bullseye
    files:
      - sysinv/sysinv/*
    vars:
@ -100,7 +82,7 @@
    parent: tox
    description: |
      Run pylint test for sysinv
-    nodeset: ubuntu-bionic
+    nodeset: debian-bullseye
    required-projects:
      - starlingx/fault
      - starlingx/utilities
@ -116,7 +98,7 @@
    parent: tox
    description: |
      Run bandit test for sysinv
-    nodeset: ubuntu-bionic
+    nodeset: debian-bullseye
    files:
      - sysinv/sysinv/*
    vars:
--- a/sysinv/sysinv/sysinv/pylint.rc
+++ b/sysinv/sysinv/sysinv/pylint.rc
@ -31,97 +31,13 @@ extension-pkg-whitelist=lxml.etree,greenlet


 [MESSAGES CONTROL]
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time.
-#
-# Python3 checker:
-#
-# E1601: print-statement
-# E1602: parameter-unpacking
-# E1603: unpacking-in-except
-# E1604: old-raise-syntax
-# E1605: backtick
-# E1606: long-suffix
-# E1607: old-ne-operator
-# E1608: old-octal-literal
-# E1609: import-star-module-level
-# E1610: non-ascii-bytes-literal
-# E1611: invalid-unicode-literal
-# W1601: apply-builtin
-# W1602: basestring-builtin
-# W1603: buffer-builtin
-# W1604: cmp-builtin
-# W1605: coerce-builtin
-# W1606: execfile-builtin
-# W1607: file-builtin
-# W1608: long-builtin
-# W1609: raw_input-builtin
-# W1610: reduce-builtin
-# W1611: standarderror-builtin
-# W1612: unicode-builtin
-# W1613: xrange-builtin
-# W1614: coerce-method
-# W1615: delslice-method
-# W1616: getslice-method
-# W1617: setslice-method
-# W1618: no-absolute-import
-# W1619: old-division
-# W1620: dict-iter-method
-# W1621: dict-view-method
-# W1622: next-method-called
-# W1623: metaclass-assignment
-# W1624: indexing-exception
-# W1625: raising-string
-# W1626: reload-builtin
-# W1627: oct-method
-# W1628: hex-method
-# W1629: nonzero-method
-# W1630: cmp-method
-# W1632: input-builtin
-# W1633: round-builtin
-# W1634: intern-builtin
-# W1635: unichr-builtin
-# W1636: map-builtin-not-iterating
-# W1637: zip-builtin-not-iterating
-# W1638: range-builtin-not-iterating
-# W1639: filter-builtin-not-iterating
-# W1640: using-cmp-argument
-# W1641: eq-without-hash
-# W1642: div-method
-# W1643: idiv-method
-# W1644: rdiv-method
-# W1645: exception-message-attribute
-# W1646: invalid-str-codec
-# W1647: sys-max-int
-# W1648: bad-python3-import
-# W1649: deprecated-string-function
-# W1650: deprecated-str-translate-call
-# W1651: deprecated-itertools-function
-# W1652: deprecated-types-field
-# W1653: next-method-defined
-# W1654: dict-items-not-iterating
-# W1655: dict-keys-not-iterating
-# W1656: dict-values-not-iterating
-# W1657: deprecated-operator-function
-# W1658: deprecated-urllib-function
-# W1659: xreadlines-attribute
-# W1660: deprecated-sys-function
-# W1661: exception-escape
-# W1662: comprehension-escape
-enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
-       W1651,W1649,W1657,W1660,W1658,W1659,W1623,W1622,W1620,W1621,W1645,W1641,
-       W1624,W1648,W1625,W1611,W1662,W1661,W1650,W1640,W1630,W1614,W1615,W1642,
-       W1616,W1628,W1643,W1629,W1627,W1644,W1617,W1601,W1602,W1603,W1604,W1605,
-       W1654,W1655,W1656,W1619,W1606,W1607,W1639,W1618,W1632,W1634,W1608,W1636,
-       W1653,W1646,W1638,W1609,W1610,W1626,W1633,W1647,W1635,W1612,W1613,W1637
-
 # Disable the message, report, category or checker with the given id(s). You
 # can either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once).
 # See "Messages Control" section of
 # https://pylint.readthedocs.io/en/latest/user_guide
+
 # We are disabling (C)onvention
 # We are disabling (R)efactor
 # We are selectively disabling (W)arning
@ -133,8 +49,11 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
 # W0106: expression-not-assigned
 # W0107: unnecessary-pass
 # W0108: unnecessary-lambda
-# W0110: deprecated-lambda
+# W0109: duplicate-key                  !!!
 # W0123: eval-used
+# W0125: using-constant-test            !!!
+# W0133: pointless-exception-statement  !!!
+# W0143: comparison-with-callable       !!!
 # W0150: lost-exception
 # W0201: attribute-defined-outside-init
 # W0211: bad-staticmethod-argument
@ -143,10 +62,11 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
 # W0223: abstract-method
 # W0231: super-init-not-called
 # W0235: useless-super-delegation
+# W0237: arguments-renamed  !!!
 # W0311: bad-indentation
 # W0402: deprecated-module
-# W0403: relative-import
 # W0404: reimported
+# W0602: global-variable-not-assigned  !!!
 # W0603: global-statement
 # W0612: unused-variable
 # W0613: unused-argument
@ -154,34 +74,46 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
 # W0622: redefined-builtin
 # W0631: undefined-loop-variable
 # W0632: unbalanced-tuple-unpacking
-# W0703: broad-except
+# W0703: broad-except (pylint 2.16 renamed to broad-except-caught)
 # W0706: try-except-raise
+# W0707: raise-missing-from
+# W0719: broad-exception-raised
 # W1113: keyword-arg-before-vararg
 # W1201: logging-not-lazy
+# W1310: format-string-without-interpolation  !!!
 # W1401: anomalous-backslash-in-string
+# W1406: redundant-u-string-prefix
 # W1501: subprocess-popen-preexec-fn
 # W1505: deprecated-method
 # W1509: subprocess-popen-preexec-fn
+# W1514: unspecified-encoding
 # W1618: no-absolute-import
+# W3101: missing-timeout
 # All these errors should be fixed:
+# E0601: used-before-assignment  !!!
+# E0605: invalid-all-format      !!!
 # E1101: no-member
 # E1111: assignment-from-no-return
-disable=C, R, fixme, W0101, W0105, W0106, W0107, W0108, W0110, W0123, W0150,
-        W0201, W0211, W0212, W0221, W0223, W0231, W0235, W0311, W0402, W0403,
-        W0404, W0603, W0612, W0613, W0621, W0622, W0631, W0632, W0701, W0703,
-        W0706, W1113, W1201, W1401, W1505, W1509, W1618,
-        E1101, E1111
+# E1120: no-value-for-parameter  !!!
+# E1121: too-many-function-args  !!!
+# E1123: unexpected-keyword-arg  !!!
+# E1136: unsubscriptable-object  !!!
+disable=C, R, fixme,
+        W0101, W0105, W0106, W0107, W0108, W0109, W0123,
+        W0125, W0133, W0143, W0150, W0201, W0211, W0212,
+        W0221, W0223, W0231, W0235, W0237, W0311, W0402,
+        W0404, W0602, W0603, W0612, W0613, W0621, W0622,
+        W0631, W0632, W0701, W0703, W0706, W0707, W0719,
+        W1113, W1201, W1310, W1401, W1406, W1505, W1509,
+        W1514, W1618, W3101,
+        E0601, E0605, E1101, E1111, E1120, E1121, E1123,
+        E1136

 [REPORTS]
 # Set the output format. Available formats are text, parseable, colorized, msvs
 # (visual studio) and html
 output-format=text

-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
 # Tells whether to display a full report or only the messages
 reports=yes

@ -240,9 +172,6 @@ generated-members=REQUEST,acl_users,aq_parent


 [BASIC]
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
 # Regular expression which should only match correct module names
 module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$

@ -332,7 +261,7 @@ max-locals=15
 max-returns=6

 # Maximum number of branch for function / method body
-max-branchs=12
+max-branches=12

 # Maximum number of statements in function / method body
 max-statements=50
@ -359,6 +288,6 @@ valid-classmethod-first-arg=cls


 [EXCEPTIONS]
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions=builtins.BaseException,builtins.Exception
+
--- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py
+++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py
@ -61,7 +61,7 @@ def delete_k8s_configmap(parameter, kube_operator):
    ------
    wsme.exc.ClientSideError
    """
-    _volume, _ = service_parameter.parse_volume_string_to_dict(parameter)
+    _volume, _noConfigMap = service_parameter.parse_volume_string_to_dict(parameter)

    # only delete configmaps for 'File' type since
    # 'DirectoryorCreate' type has no associated configmaps
@ -160,7 +160,7 @@ def update_k8s_configmap(parameter, kube_operator):
    ------
    wsme.exc.ClientSideError
    """
-    _volume, _ = service_parameter.parse_volume_string_to_dict(parameter)
+    _volume, _noConfigMap = service_parameter.parse_volume_string_to_dict(parameter)

    # only update configmaps for 'File' type since
    # 'DirectoryorCreate' type has no associated configmaps
--- a/sysinv/sysinv/sysinv/test-requirements.txt
+++ b/sysinv/sysinv/sysinv/test-requirements.txt
@ -2,7 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 hacking>=1.1.0,<=2.0.0 # Apache-2.0
-astroid <= 2.2.5
+astroid
 bandit<1.7.2;python_version>="3.0"
 coverage>=3.6
 fixtures>=3.0.0 # Apache-2.0/BSD
@ -16,6 +16,5 @@ stestr>=1.0.0 # Apache-2.0
 testrepository>=0.0.18
 testtools!=1.2.0,>=0.9.36
 isort<5;python_version>="3.0"
-pylint<2.1.0;python_version<"3.0" # GPLv2
-pylint<2.4.0;python_version>="3.0" # GPLv2
+pylint
 pycryptodomex
--- a/sysinv/sysinv/sysinv/tox.ini
+++ b/sysinv/sysinv/sysinv/tox.ini
@ -1,5 +1,5 @@
 [tox]
-envlist = flake8,py27,py36,py39,pylint
+envlist = flake8,py39,pylint
 minversion = 1.6
 skipsdist = True

@ -10,7 +10,7 @@ stxdir = {toxinidir}/../../../..
 distshare={toxworkdir}/.tox/distshare

 [testenv]
-basepython = python3
+basepython = python3.9
 usedevelop = True

 # tox is silly... these need to be separated by a newline....
@ -18,7 +18,7 @@ allowlist_externals = bash
                      find

 install_command = pip install -v -v -v \
-    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
    {opts} {packages}

 # Note the hash seed is set to 0 until can be tested with a
@ -42,7 +42,7 @@ deps = -r{toxinidir}/requirements.txt
       -e{[tox]stxdir}/fault/fm-api/source
       -e{[tox]stxdir}/fault/python-fmclient/fmclient
       -e{[tox]stxdir}/config/controllerconfig/controllerconfig
-       -e{[tox]stxdir}/update/cgcs-patch/cgcs-patch
+       -e{[tox]stxdir}/update/sw-patch/cgcs-patch
       -e{[tox]stxdir}/utilities/utilities/platform-util/platform-util
       -e{[tox]stxdir}/utilities/ceph/python-cephclient/python-cephclient
       -e{[tox]stxdir}/config/sysinv/cgts-client/cgts-client
@ -95,42 +95,19 @@ exclude = build,dist,tools,.eggs
 max-line-length=120

 [testenv:flake8]
-basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
 commands =
  flake8 {posargs} . \
    scripts/manage-partitions \
    scripts/query_pci_id

-[testenv:py27]
-basepython = python2.7
-passenv = CURL_CA_BUNDLE
-commands =
-  {[testenv]commands}
-  stestr run {posargs}
-  stestr slowest
-
-[testenv:py36]
-basepython = python3.6
-commands =
-  {[testenv]commands}
-  stestr run {posargs}
-  stestr slowest
-
 [testenv:py39]
-basepython = python3.9
-
-install_command = pip install -v -v -v \
-    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
-    {opts} {packages}
 commands =
  stestr run {posargs}
  stestr slowest

-
 [testenv:pep8]
 # testenv:flake8 clone
-basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
 commands = {[testenv:flake8]commands}

@ -170,20 +147,18 @@ skips = B101,B103,B104,B105,B108,B110,B303,B307,B310,B311,B314,B318,B320,B404,B4
 exclude = tests

 [testenv:bandit]
-basepython = python3.6
 deps = -r{toxinidir}/test-requirements.txt
 commands = bandit --ini tox.ini -n 5 -r sysinv

 [testenv:pylint]
-basepython = python3.6
+install_command = pip install -v -v -v \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
+    {opts} {packages}
 commands =
     pylint {posargs} sysinv --rcfile=./pylint.rc

 [testenv:cover]
-basepython = python3.9
-install_command = pip install -v -v -v \
-    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
-    {opts} {packages}
+# not sure is passenv is still needed
 passenv = CURL_CA_BUNDLE
 deps = {[testenv]deps}
 setenv = {[testenv]setenv}