Update app Zuul Check Jobs.

Modify code to conform to flake8 and pylint.

Jobs are now flake8, pylint, py39 and metadata.

Test Plan
PASS - All zuul jobs pass as expected.

Story: 2010929
Task: 49279

Change-Id: I7e105edccbff006423ada7ab1f99449822dd09bc
Signed-off-by: Reed, Joshua <Joshua.Reed@windriver.com>

.gitignore

@@ -1 +1,2 @@
 .tox
+venv/

.zuul.yaml

@@ -5,18 +5,22 @@
         - openstack-tox-linters
         - k8sapp-portieris-tox-py39
         - k8sapp-portieris-tox-pylint
+        - k8sapp-portieris-tox-flake8
+        - k8sapp-portieris-tox-metadata
     gate:
       jobs:
         - openstack-tox-linters
         - k8sapp-portieris-tox-py39
         - k8sapp-portieris-tox-pylint
+        - k8sapp-portieris-tox-flake8
+        - k8sapp-portieris-tox-metadata
     post:
       jobs:
         - stx-portieris-armada-app-upload-git-mirror
 
 - job:
     name: k8sapp-portieris-tox-py39
-    parent: tox-py39
+    parent: openstack-tox-py39
     description: |
         Run py39 test for portieris
     nodeset: debian-bullseye
@@ -25,12 +29,12 @@
       - starlingx/fault
       - starlingx/update
       - starlingx/utilities
+      - starlingx/root
     files:
       - python3-k8sapp-portieris/*
     vars:
-        tox_envlist: py39
-        python_version: 3.9
         tox_extra_args: -c python3-k8sapp-portieris/k8sapp_portieris/tox.ini
+        tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt'
 
 - job:
     name: k8sapp-portieris-tox-pylint
@@ -43,11 +47,51 @@
       - starlingx/fault
       - starlingx/update
       - starlingx/utilities
+      - starlingx/root
     files:
       - python3-k8sapp-portieris/*
     vars:
       tox_envlist: pylint
       tox_extra_args: -c python3-k8sapp-portieris/k8sapp_portieris/tox.ini
+      tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt'
+
+- job:
+    name: k8sapp-portieris-tox-flake8
+    parent: tox
+    description: |
+      Run flake8 test for k8sapp_portieris
+    nodeset: debian-bullseye
+    required-projects:
+      - starlingx/config
+      - starlingx/fault
+      - starlingx/update
+      - starlingx/utilities
+      - starlingx/root
+    files:
+      - python3-k8sapp-portieris/*
+    vars:
+      tox_envlist: flake8
+      tox_extra_args: -c python3-k8sapp-portieris/k8sapp_portieris/tox.ini
+      tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt'
+
+- job:
+    name: k8sapp-portieris-tox-metadata
+    parent: tox
+    description: |
+      Run metadata test for k8sapp_portieris
+    nodeset: debian-bullseye
+    required-projects:
+      - starlingx/config
+      - starlingx/fault
+      - starlingx/update
+      - starlingx/utilities
+      - starlingx/root
+    files:
+      - python3-k8sapp-portieris/*
+    vars:
+      tox_envlist: metadata
+      tox_extra_args: -c python3-k8sapp-portieris/k8sapp_portieris/tox.ini
+      tox_constraints_file: '{{ ansible_user_dir }}/src/opendev.org/starlingx/root/build-tools/requirements/debian/upper-constraints.txt'
 
 - job:
     name: stx-portieris-armada-app-upload-git-mirror

python3-k8sapp-portieris/k8sapp_portieris/pylint.rc

@@ -31,21 +31,127 @@ extension-pkg-whitelist=lxml.etree,greenlet
 
 
 [MESSAGES CONTROL]
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time.
-
 # Disable the message, report, category or checker with the given id(s). You
 # can either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once).
 # See "Messages Control" section of
 # https://pylint.readthedocs.io/en/latest/user_guide
-# We are disabling (C)onvention
-# We are disabling (R)efactor
-# W0212: protected-access
-# W1618: no-absolute-import
-disable=C, R, W0212, W1618
+disable=
+        # C codes refer to Convention
+ C0103, # invalid-name
+ C0104, # disallowed-nameA
+ C0112, # empty-docstring
+ C0114, # missing-module-docstring
+ C0115, # missing-class-docstring
+ C0116, # missing-function-docstring
+ C0123, # unidiomatic-typecheck !!!
+ C0201, # consider-iterating-dictionary
+ C0202, # bad-classmethod-argument
+ C0206, # consider-using-dict-items
+ C0207, # use-maxsplit-arg
+ C0209, # consider-using-f-string
+ C0301, # line-too-long
+ C0302, # too-many-lines
+ C0325, # superfluous-parens
+ C0411, # wrong-import-order
+ C0412, # ungrouped-imports
+ C0413, # wrong-import-position
+ C0414, # useless-import-alias  !!!
+ C0415, # import-outside-toplevel
+ C1802, # use-implicit-booleaness-not-len !!!
+ C2801, # unnecessary-dunder-call  !!!
+ C3002, # unnecessary-direct-lambda-call !!!
+        # R codes refer to refactoring
+ R0022, # useless-option-value !!!
+ R0205, # useless-object-inheritance
+ R0402, # consider-using-from-import
+ R0901, # too-many-ancestors
+ R0902, # too-many-instance-attributes
+ R0903, # too-few-public-methods
+ R0904, # too-many-public-methods
+ R0911, # too-many-return-statements
+ R0912, # too-many-branches
+ R0913, # too-many-arguments
+ R0914, # too-many-locals
+ R0915, # too-many-statements
+ R0916, # too-many-boolean-expressions
+ R1702, # too-many-nested-blocks
+ R1703, # simplifiable-if-statement
+ R1704, # redefined-argument-from-local !!!
+ R1705, # no-else-return
+ R1707, # trailing-comma-tuple  !!!
+ R1708, # stop-iteration-return !!!
+ R1710, # inconsistent-return-statements
+ R1711, # useless-return
+ R1714, # consider-using-in
+ R1717, # consider-using-dict-comprehension !!!
+ R1718, # consider-using-set-comprehension
+ R1719, # simplifiable-if-expression
+ R1720, # no-else-raise
+ R1721, # unnecessary-comprehension
+ R1722, # consider-using-sys-exit !!!
+ R1723, # no-else-break
+ R1724, # no-else-continue
+ R1725, # super-with-arguments
+ R1726, # simplifiable-condition  !!!
+ R1728, # consider-using-generator
+ R1729, # use-a-generator
+ R1730, # consider-using-min-builtin  !!!
+ R1731, # consider-using-max-builtin  !!!
+ R1732, # consider-using-with
+ R1733, # unnecessary-dict-index-lookup !!
+ R1734, # use-list-literal
+ R1735, # use-dict-literal
+        # W codes are warnings
+ W0101, # unreachable
+ W0105, # pointless-string-statement
+ W0106, # expression-not-assigned
+ W0107, # unnecessary-pass
+ W0108, # unnecessary-lambda
+ W0109, # duplicate-key                  !!!
+ W0123, # eval-used
+ W0125, # using-constant-test            !!!
+ W0133, # pointless-exception-statement  !!!
+ W0143, # comparison-with-callable       !!!
+ W0150, # lost-exception
+ W0201, # attribute-defined-outside-init
+ W0211, # bad-staticmethod-argument
+ W0212, # protected-access
+ W0221, # arguments-differ
+ W0223, # abstract-method
+ W0231, # super-init-not-called
+ W0235, # useless-super-delegation
+ W0237, # arguments-renamed  !!!
+ W0311, # bad-indentation
+ W0402, # deprecated-module
+ W0404, # reimported
+ W0511, # fixme
+ W0602, # global-variable-not-assigned  !!!
+ W0603, # global-statement
+ W0612, # unused-variable
+ W0613, # unused-argument
+ W0621, # redefined-outer-name
+ W0622, # redefined-builtin
+ W0631, # undefined-loop-variable
+ W0703, # broad-except (pylint 2.16 renamed to broad-except-caught)
+ W0706, # try-except-raise
+ W0707, # raise-missing-from
+ W0719, # broad-exception-raised
+ W1113, # keyword-arg-before-vararg
+ W1310, # format-string-without-interpolation  !!!
+ W1401, # anomalous-backslash-in-string
+ W1406, # redundant-u-string-prefix
+ W1505, # deprecated-method
+ W1514, # unspecified-encoding
+ W3101, # missing-timeout
+ E0601, # used-before-assignment  !!!
+ E0605, # invalid-all-format      !!!
+ E1101, # no-member
+ E1111, # assignment-from-no-return
+ E1121, # too-many-function-args  !!!
+ E1123, # unexpected-keyword-arg  !!!
+ E1136, # unsubscriptable-object  !!!
 
 [REPORTS]
 # Set the output format. Available formats are text, parseable, colorized, msvs
@@ -53,7 +159,7 @@ disable=C, R, W0212, W1618
 output-format=text
 
 # Tells whether to display a full report or only the messages
-reports=yes
+reports=no
 
 # Python expression which should return a note less than 10 (10 is the highest
 # note). You have access to the variables errors warning, statement which
@@ -226,6 +332,5 @@ valid-classmethod-first-arg=cls
 
 
 [EXCEPTIONS]
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=builtins.Exception
+# Exceptions that will emit a warning when caught.
+overgeneral-exceptions=builtins.BaseException,builtins.Exception

python3-k8sapp-portieris/k8sapp_portieris/requirements.txt

@@ -1 +1,2 @@
 pbr>=2.0.0
+PyYAML>=3.10.0

python3-k8sapp-portieris/k8sapp_portieris/test-requirements.txt

@@ -2,21 +2,19 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 hacking>=1.1.0,<=2.0.0 # Apache-2.0
+astroid
+bandit<1.7.2;python_version>="3.0"
 coverage>=3.6
-discover
+fixtures>=3.0.0 # Apache-2.0/BSD
 mock>=2.0.0 # BSD
-passlib>=1.7.0
-psycopg2-binary
 python-subunit>=0.0.18
 requests-mock>=0.6.0 # Apache-2.0
+sphinx
+oslosphinx
 oslotest>=3.2.0 # Apache-2.0
 stestr>=1.0.0 # Apache-2.0
+testrepository>=0.0.18
 testtools!=1.2.0,>=0.9.36
-ipaddr
-pytest
-pyudev
-migrate
-markupsafe
-bandit
-flake8-bugbear
+isort<5;python_version>="3.0"
 pylint
+pycryptodomex

python3-k8sapp-portieris/k8sapp_portieris/tox.ini

@@ -1,20 +1,24 @@
 [tox]
-envlist = flake8,py39,pylint,bandit,cover
-minversion = 2.3
+envlist = flake8,py39,pylint,metadata
+minversion = 1.6
+skipsdist = True
 
 # tox does not work if the path to the workdir is too long, so move it to /tmp
+# tox 3.1.0 adds TOX_LIMITED_SHEBANG
 toxworkdir = /tmp/{env:USER}_k8sportieristox
 stxdir = {toxinidir}/../../..
 distshare={toxworkdir}/.tox/distshare
 
 [testenv]
+basepython = python3.9
+usedevelop = True
+
+# tox is silly... these need to be separated by a newline....
 allowlist_externals = bash
                       find
-basepython = python3
-sitepackages = False
+                      echo
 
 install_command = pip install -v -v -v \
-    -c{toxinidir}/upper-constraints.txt \
     -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
     {opts} {packages}
 
@@ -22,6 +26,7 @@ install_command = pip install -v -v -v \
 # random hash seed successfully.
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
+         PIP_RESOLVER_DEBUG=1
          PYTHONDONTWRITEBYTECODE=1
          OS_TEST_PATH=./k8sapp_portieris/tests
          LANG=en_US.UTF-8
@@ -32,56 +37,126 @@ setenv = VIRTUAL_ENV={envdir}
          TOX_WORK_DIR={toxworkdir}
          PYLINTHOME={toxworkdir}
 
-# the path to cgcs-patch needs to be updated to sw-patch
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
        -e{[tox]stxdir}/config/sysinv/sysinv/sysinv
        -e{[tox]stxdir}/config/tsconfig/tsconfig
        -e{[tox]stxdir}/fault/fm-api/source
        -e{[tox]stxdir}/fault/python-fmclient/fmclient
-       -e{[tox]stxdir}/utilities/ceph/python-cephclient/python-cephclient
        -e{[tox]stxdir}/update/sw-patch/cgcs-patch
-
+       -e{[tox]stxdir}/utilities/ceph/python-cephclient/python-cephclient
 
 commands =
   find . -type f -name "*.pyc" -delete
 
 [flake8]
+# H series are hacking
+# H101 is TODO
+# H102 is apache license
+# H104 file contains only comments (ie: license)
+# H105 author tags
+# H306 imports not in alphabetical order
+# H401 docstring should not start with a space
+# H403 multi line docstrings should end on a new line
+# H404 multi line docstring should start without a leading new line
+# H405 multi line docstring summary not separated with an empty line
+# H701 Empty localization string
+# H702 Formatting operation should be outside of localization method call
+# H703 Multiple positional placeholders
+
+# B series are bugbear
+# B006 Do not use mutable data structures for argument defaults. Needs to be FIXED.
+# B007 Loop control variable not used within the loop body.
+# B009 Do not call getattr with a constant attribute value
+# B010 Do not call setattr with a constant attribute value
+# B012 return/continue/break inside finally blocks cause exceptions to be silenced
+# B014 Redundant exception types
+# B301 Python 3 does not include `.iter*` methods on dictionaries. (this should be suppressed on a per line basis)
+
+# W series are warnings
+# W503 line break before binary operator
+# W504 line break after binary operator
+# W605 invalid escape sequence
+
+# E series are pep8
+# E117 over-indented
+# E126 continuation line over-indented for hanging indent
+# E127 continuation line over-indented for visual indent
+# E128 continuation line under-indented for visual indent
+# E402 module level import not at top of file
+# E741 ambiguous variable name
+
+ignore = H101,H102,H104,H105,H306,H401,H403,H404,H405,H701,H702,H703,
+         B006,B007,B009,B010,B012,B014,B301
+         W503,W504,W605,
+         E117,E126,E127,E128,E402,E741
 exclude = build,dist,tools,.eggs
 max-line-length=120
 
 [testenv:flake8]
 deps = -r{toxinidir}/test-requirements.txt
 commands =
-  flake8 {posargs} .
+  flake8 {posargs} ./k8sapp_portieris
 
 [testenv:py39]
-basepython = python3.9
 commands =
-  {[testenv]commands}
   stestr run {posargs}
   stestr slowest
 
 [testenv:pep8]
-deps = {[testenv:flake8]deps}
+# testenv:flake8 clone
+deps = -r{toxinidir}/test-requirements.txt
 commands = {[testenv:flake8]commands}
 
 [testenv:venv]
 commands = {posargs}
 
 [bandit]
-# Add bandit configuration here
+# The following bandit tests are being skipped:
+# B101: Test for use of assert
+# B103: Test for setting permissive file permissions
+# B104: Test for binding to all interfaces
+# B105: Test for use of hard-coded password strings
+# B108: Test for insecure usage of tmp file/directory
+# B110: Try, Except, Pass detected.
+# B303: Use of insecure MD2, MD4, MD5, or SHA1 hash function.
+# B307: Blacklisted call to eval.
+# B310: Audit url open for permitted schemes
+# B311: Standard pseudo-random generators are not suitable for security/cryptographic purposes
+# B314: Blacklisted calls to xml.etree.ElementTree
+# B318: Blacklisted calls to xml.dom.minidom
+# B320: Blacklisted calls to lxml.etree
+# B404: Import of subprocess module
+# B405: import xml.etree
+# B408: import xml.minidom
+# B410: import lxml
+# B506: Test for use of yaml load
+# B602: Test for use of popen with shell equals true
+# B603: Test for use of subprocess without shell equals true
+# B604: Test for any function with shell equals true
+# B605: Test for starting a process with a shell
+# B607: Test for starting a process with a partial path
+# B608: Possible SQL injection vector through string-based query
+#
+# Note: 'skips' entry cannot be split across multiple lines
+#
+skips = B101,B103,B104,B105,B108,B110,B303,B307,B310,B311,B314,B318,B320,B404,B405,B408,B410,B506,B602,B603,B604,B605,B607,B608
+exclude = tests
 
 [testenv:bandit]
 deps = -r{toxinidir}/test-requirements.txt
 commands = bandit --ini tox.ini -n 5 -r k8sapp_portieris
 
 [testenv:pylint]
-deps = {[testenv]deps}
+install_command = pip install -v -v -v \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
+    {opts} {packages}
 commands =
      pylint {posargs} k8sapp_portieris --rcfile=./pylint.rc
 
 [testenv:cover]
+# not sure is passenv is still needed
+passenv = CURL_CA_BUNDLE
 deps = {[testenv]deps}
 setenv = {[testenv]setenv}
          PYTHON=coverage run --parallel-mode
@@ -103,3 +178,11 @@ commands =
 deps = pip_missing_reqs
        -rrequirements.txt
 commands=pip-missing-reqs -d --ignore-file=/k8sapp_portieris/tests k8sapp_portieris
+
+[testenv:metadata]
+install_command = pip install -v -v -v \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
+    {opts} {packages}
+# Pass top level app folder to 'sysinv-app tox' command.
+commands =
+  bash -c "echo $(dirname $(dirname $(pwd))) | xargs -n 1 sysinv-app tox"

stx-portieris-helm/debian/deb_folder/rules

@@ -40,9 +40,9 @@ override_dh_auto_build:
 	cp $(HELM_FOLDER)/portieris*.tgz $(STAGING)/charts
 
 	# Populate metadata.
-	sed -i 's/@APP_NAME@/$(APP_NAME)/g' $(STAGING)/metadata.yaml
-	sed -i 's/@APP_VERSION@/$(APP_VERSION)/g' $(STAGING)/metadata.yaml
-	sed -i 's/@HELM_REPO@/$(HELM_REPO)/g' $(STAGING)/metadata.yaml
+	sed -i 's/APP_REPLACE_NAME/$(APP_NAME)/g' $(STAGING)/metadata.yaml
+	sed -i 's/APP_REPLACE_VERSION/$(APP_VERSION)/g' $(STAGING)/metadata.yaml
+	sed -i 's/HELM_REPLACE_REPO/$(HELM_REPO)/g' $(STAGING)/metadata.yaml
 
 	# Copy the plugins: installed in the buildroot
 	mkdir -p $(STAGING)/plugins

stx-portieris-helm/stx-portieris-helm/files/metadata.yaml

@@ -1,7 +1,7 @@
+app_name: APP_REPLACE_NAME
+app_version: APP_REPLACE_VERSION
+helm_repo: HELM_REPLACE_REPO
+
 maintain_user_overrides: true
 upgrades:
   auto_update: true
-
-app_name: @APP_NAME@
-app_version: @APP_VERSION@
-helm_repo: @HELM_REPO@

test-requirements.txt

@@ -1,3 +1,3 @@
 # hacking pulls in flake8
-hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
+hacking>=1.1.0,<=2.0.0 # Apache-2.0
 bashate >= 0.2

tox.ini

@@ -1,11 +1,14 @@
 [tox]
 envlist = linters
-minversion = 2.3
+minversion = 2.9
 skipsdist = True
 sitepackages=False
 
 [testenv]
-install_command = pip install -U {opts} {packages}
+basepython = python3
+install_command = pip install -U \
+    {opts} {packages} \
+    -c{env:TOX_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt}
 setenv =
    VIRTUAL_ENV={envdir}
    OS_STDOUT_CAPTURE=1
@@ -18,6 +21,22 @@ deps =
 allowlist_externals =
   bash
 
+[testenv:flake8]
+basepython = python3
+description = Dummy environment to allow flake8 to be run in subdir tox
+
+[testenv:pylint]
+basepython = python3
+description = Dummy environment to allow pylint to be run in subdir tox
+
+[testenv:metadata]
+basepython = python3
+description = Dummy environment to allow sysinv-app to be run in subdir tox
+
+[testenv:bandit]
+basepython = python3
+description = Dummy environment to allow bandit to be run in subdir tox
+
 [testenv:bashate]
 # Treat all E* codes as Errors rather than warnings using: -e 'E*'
 commands =
@@ -33,8 +52,3 @@ commands =
 [testenv:linters]
 commands =
     {[testenv:bashate]commands}
-
-[testenv:pylint]
-basepython = python3
-description = Dummy environment to allow pylint to be run in subdir tox
-