[WIP] Update molteniron tests and requirements

Change-Id: I22777cf34888171daf69c44705d8ecc7339f8278

.gitignore

@@ -0,0 +1,61 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg*
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+cover/
+.coverage*
+!.coveragerc
+.tox
+nosetests.xml
+.stestr/
+.venv
+testenv
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+doc/build
+doc/source/reference/api
+
+# pbr generates these
+AUTHORS
+ChangeLog
+
+# Editors
+*~
+.*.swp
+.*sw?
+
+# Files created by releasenotes build
+releasenotes/build
+

.zuul.yaml

@@ -1,4 +1,8 @@
 - project:
     templates:
-      - openstack-python3-ussuri-jobs
       - check-requirements
+      - openstack-cover-jobs
+      - openstack-lower-constraints-master-branch-jobs
+      - openstack-python3-yoga-jobs
+      - publish-openstack-docs-pti
+      - release-notes-jobs-python3

doc/requirements.txt

@@ -0,0 +1,7 @@
+openstackdocstheme>=2.2.0 # Apache-2.0
+os-api-ref>=1.4.0 # Apache-2.0
+reno>=3.1.0 # Apache-2.0
+sphinx>=2.0.0,!=2.1.0 # BSD
+sphinxcontrib-apidoc>=0.2.0  # BSD
+sphinxcontrib-seqdiag>=0.8.4 # BSD
+sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD

lower-constraints.txt

@@ -0,0 +1,6 @@
+PyMySQL==0.8.0
+SQLAlchemy-Utils==0.30.11
+SQLAlchemy==1.2.19
+daemonize==2.5.0
+pbr==2.0.0
+PyYAML==3.10

molteniron/molteniron.py

@@ -45,6 +45,7 @@ def makeRegistrar():
     registrar.all = registry
     return registrar
 
+
 # Create the decorator
 command = makeRegistrar()
 

molteniron/moltenirond.py

@@ -29,17 +29,16 @@ This is the MoltenIron server.
 
 import argparse
 import calendar
+import collections
+from contextlib import contextmanager
 from datetime import datetime
 import json
 import os
-from pkg_resources import resource_filename
 import sys
 import time
 import traceback
-import yaml
-
-from contextlib import contextmanager
 
+from pkg_resources import resource_filename
 from sqlalchemy import create_engine
 from sqlalchemy import Column, Integer, String, ForeignKey
 from sqlalchemy.exc import InternalError, OperationalError
@@ -49,10 +48,8 @@ from sqlalchemy.schema import MetaData, Table
 from sqlalchemy.sql import insert, update, delete
 from sqlalchemy.sql import and_
 from sqlalchemy.types import TIMESTAMP
-
 import sqlalchemy_utils
-
-import collections  # noqa
+import yaml
 
 if sys.version_info >= (3, 0):
     from http.server import HTTPServer, BaseHTTPRequestHandler  # noqa
@@ -229,8 +226,8 @@ class Nodes(declarative_base()):
         """Returns a map of the database row contents"""
         return {key: value for key, value
                 in list(self.__dict__.items())
-                if not key.startswith('_') and
-                not isinstance(key, collections.Callable)}
+                if not key.startswith('_')
+                and not isinstance(key, collections.Callable)}
 
     def __repr__(self):
         fmt = """<Node(name='%s',
@@ -291,6 +288,7 @@ ip='%s' />"""
                       self.node_id,
                       self.ip)
 
+
 TYPE_MYSQL = 1
 # Is there a mysql memory path?
 TYPE_SQLITE = 3
@@ -1039,10 +1037,7 @@ class DataBase(object):
         for (field, length, _, skip) in ei:
             if skip:
                 continue
-            dl += (" " +
-                   field +
-                   ' ' * (length - len(field)) +
-                   " +")
+            dl += (" " + field + ' ' * (length - len(field)) + " +")
 
         index = 0
         fl = "|"

molteniron/tests/testAddBMNode.py

@@ -22,12 +22,15 @@ Tests the addBMNode MoltenIron command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")
     parser.add_argument("-c",

molteniron/tests/testAllocateBM.py

@@ -23,12 +23,14 @@ Tests the MoltenIron allocateBM command.
 
 import argparse
 import json
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 def result_to_r(res):
     """Takes a result and returns the request parameters."""

molteniron/tests/testCull.py

@@ -23,13 +23,15 @@ Tests the MoltenIron cull command.
 
 import argparse
 import json
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
 import time
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 def result_to_r(res):
     """Takes a result and returns the request parameters."""
@@ -68,6 +70,7 @@ def compare_culled_nodes(lhs, rhs_r, rhs_n):
     assert lhs_r == rhs_r
     assert lhs_n == rhs_n
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")
     parser.add_argument("-c",

molteniron/tests/testDeallocateBM.py

@@ -23,12 +23,14 @@ Tests the MoltenIron deallocateBM command.
 
 import argparse
 import json
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 def result_to_r(res):
     """Takes a result and returns the request parameters."""

molteniron/tests/testDeallocateOwner.py

@@ -23,12 +23,14 @@ Tests the MoltenIron deallocateOwner command.
 
 import argparse
 import json
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 def result_to_r(res):
     """Takes a result and returns the request parameters."""

molteniron/tests/testDoClean.py

@@ -22,12 +22,14 @@ Tests the MoltenIron doClean command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")

molteniron/tests/testGetField.py

@@ -22,12 +22,14 @@ Tests the MoltenIron get_field command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")

molteniron/tests/testGetIps.py

@@ -22,12 +22,14 @@ Tests the MoltenIron get_ips command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")

molteniron/tests/testRemoveBMNode.py

@@ -22,12 +22,14 @@ Tests the MoltenIron removeBMNode command.
 # pylint: disable-msg=C0103
 
 import argparse
-from molteniron import moltenirond
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import moltenirond
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Molteniron CLI tool")

requirements.txt

@@ -2,10 +2,9 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-pbr>=1.6 # Apache-2.0
-
-daemonize
-PyMySQL>=0.6.2,!=0.7.7 # MIT License
-pyyaml
-sqlalchemy
-sqlalchemy_utils
+pbr>=2.0.0 # Apache-2.0
+daemonize>=2.5.0 # MIT License
+PyMySQL>=0.8.0 # MIT License
+PyYAML>=3.10 # MIT License
+SQLAlchemy>=1.2.19 # MIT License
+SQLAlchemy-Utils>=0.30.11 # BSD

test-requirements.txt

@@ -2,16 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking>=0.11.0,<0.12 # Apache-2.0
-
-coverage>=4.0 # Apache-2.0
-python-subunit>=0.0.18 # Apache-2.0/BSD
-sphinx>=1.2.1,!=1.3b1,<1.4 # BSD
-openstackdocstheme>=1.11.0 # Apache-2.0
-oslotest>=1.10.0 # Apache-2.0
-testrepository>=0.0.18 # Apache-2.0/BSD
-testscenarios>=0.4 # Apache-2.0/BSD
-testtools>=1.4.0 # MIT
-
-# releasenotes
-reno>=1.8.0 # Apache2
+# unit tests
+coverage!=4.4,>=4.0 # Apache-2.0
+oslotest>=3.2.0 # Apache-2.0
+stestr>=2.0.0 # Apache-2.0

tox.ini

@@ -1,8 +1,8 @@
 [tox]
-minversion = 3.1.0
-envlist = py3,pypy,pep8
-ignore_basepython_conflict = True
+minversion = 3.18.0
 skipsdist = True
+envlist = py3,pep8
+ignore_basepython_conflict=true
 
 [testenv]
 basepython = python3
@@ -11,12 +11,11 @@ setenv =
    VIRTUAL_ENV={envdir}
    PYTHONWARNINGS=default::DeprecationWarning
 deps =
-  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/test-requirements.txt
-# Don't worry about installing bash commands in the virtual environment.
-whitelist_externals = mkdir
+  -r{toxinidir}/requirements.txt
+allowlist_externals = mkdir
                       diff
-#commands = python setup.py test --slowest --testr-args='{posargs}'
 commands = mkdir -p testenv/var/run/
            mkdir -p testenv/tmp/
            python setup.py \
@@ -87,19 +86,54 @@ commands = mkdir -p testenv/var/run/
                stop
 
 [testenv:pep8]
+deps=
+    hacking>=4.1.0,<5.0.0 # Apache-2.0
+    flake8-import-order>=0.17.1 # LGPLv3
+    pycodestyle>=2.0.0,<3.0.0 # MIT
 commands = flake8 {posargs}
 
 [testenv:venv]
+setenv = PYTHONHASHSEED=0
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/doc/requirements.txt
 commands = {posargs}
 
 [testenv:cover]
-commands = python setup.py test --coverage --testr-args='{posargs}'
+setenv =
+   {[testenv]setenv}
+   PYTHON=coverage run --parallel-mode
+# After running this target, visit molteniron/cover/index.html
+# in your browser, to see a nicer presentation report with annotated
+# HTML listings detailing missed lines.
+commands = coverage erase
+           stestr run {posargs}
+           coverage combine
+           coverage report
+           coverage html
+           coverage xml -o cover/coverage.xml
 
 [testenv:docs]
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/doc/requirements.txt
+commands = sphinx-build -b html -W doc/source doc/build/html
+
+[testenv:pdf-docs]
+allowlist_externals = make
+deps = {[testenv:docs]deps}
 commands =
-  sphinx-build -W -d doc/build/doctrees -b html doc/source doc/build/html
+  sphinx-build -W -b latex doc/source doc/build/pdf
+  make -C doc/build/pdf
 
 [testenv:releasenotes]
+usedevelop = False
+deps =
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -r{toxinidir}/doc/requirements.txt
 commands =
   sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
 
@@ -107,9 +141,26 @@ commands =
 commands = oslo_debug_helper {posargs}
 
 [flake8]
-# E123, E125 skipped as they are invalid PEP-8.
-
 show-source = True
-ignore = E123,E125
+# E123, E125 skipped as they are invalid PEP-8.
+# E741 ambiguous variable name.
+# W503 Line break occurred before a binary operator. Conflicts with W504.
+ignore = E123,E125,E741,W503
+# [H106] Don't put vim configuration in source files.
+# [H203] Use assertIs(Not)None to check for None.
+# [H204] Use assert(Not)Equal to check for equality.
+# [H205] Use assert(Greater|Less)(Equal) for comparison.
+# [H210] Require 'autospec', 'spec', or 'spec_set' in mock.patch/mock.patch.object calls
+# [H904] Delay string interpolations at logging calls.
+enable-extensions=H106,H203,H204,H205,H210,H904
 builtins = _
-exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,testenv
+import-order-style = pep8
+application-import-names = molteniron
+filename = *.py
+
+[testenv:lower-constraints]
+deps =
+  -c{toxinidir}/lower-constraints.txt
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/requirements.txt

utils/createDB.py

@@ -23,6 +23,7 @@ Create the MoltenIron user in mysql and grant it access.
 
 import os
 import sys
+
 import yaml
 
 
@@ -42,19 +43,19 @@ def main():
     conf = yaml.load(fobj)
 
     # Create the SQL User
-    SQL("CREATE USER '" +
-        conf["sqlUser"] +
-        "'@'localhost' IDENTIFIED BY '" +
-        conf["sqlPass"] +
-        "';")
+    SQL("CREATE USER '"
+        + conf["sqlUser"]
+        + "'@'localhost' IDENTIFIED BY '"
+        + conf["sqlPass"] + "';")
 
     # And grant that SQL user access to the MoltenIron database
-    SQL("GRANT ALL ON MoltenIron.* TO '" +
-        conf["sqlUser"] +
-        "'@'localhost';")
+    SQL("GRANT ALL ON MoltenIron.* TO '"
+        + conf["sqlUser"]
+        + "'@'localhost';")
 
     return 0
 
+
 if __name__ == "__main__":
     rc = main()
 

utils/test_hook_mi_ipmiblob.py

@@ -25,12 +25,14 @@ a MoltenIron server.
 
 import argparse
 import json
-from molteniron import molteniron
 import os
-from pkg_resources import resource_filename
 import sys
+
+from pkg_resources import resource_filename
 import yaml
 
+from molteniron import molteniron
+
 
 if __name__ == "__main__":
     mi = molteniron.MoltenIron()
@@ -107,8 +109,8 @@ if __name__ == "__main__":
         try:
             rc = response_map["status"]
         except KeyError:
-            msg = ("Error: Server returned: %s and we expected a status " +
-                   "somewhere") % (response_map, )
+            msg = ("Error: Server returned: %s and we expected a status "
+                   + "somewhere") % (response_map, )
             print(msg, file=sys.stderr)
             exit(444)
 
@@ -150,10 +152,10 @@ if __name__ == "__main__":
 
         with open(args.hardware_info, "w") as hi_obj:
             # Write one line
-            hi_obj.write(("%(ipmi_ip)s" % node) +
-                         (" %(port_hwaddr)s" +
-                          " %(ipmi_user)s" +
-                          " %(ipmi_password)s\n") % blob)
+            hi_obj.write(("%(ipmi_ip)s" % node)
+                         + (" %(port_hwaddr)s"
+                            + " %(ipmi_user)s"
+                            + " %(ipmi_password)s\n") % blob)
 
         pool = node["allocation_pool"].split(",")
 
@@ -162,8 +164,8 @@ if __name__ == "__main__":
 
         with open(args.localrc, "a") as l_obj:
             # Write multiple lines
-            l_obj.write(("IRONIC_HW_ARCH=%(cpu_arch)s\n" +
-                         "IRONIC_HW_NODE_CPU=%(cpus)s\n" +
-                         "IRONIC_HW_NODE_RAM=%(ram_mb)s\n" +
-                         "IRONIC_HW_NODE_DISK=%(disk_gb)s\n") % blob +
-                        "ALLOCATION_POOL=\"%s\"\n" % (allocation_pool, ))
+            l_obj.write(("IRONIC_HW_ARCH=%(cpu_arch)s\n"
+                         + "IRONIC_HW_NODE_CPU=%(cpus)s\n"
+                         + "IRONIC_HW_NODE_RAM=%(ram_mb)s\n"
+                         + "IRONIC_HW_NODE_DISK=%(disk_gb)s\n") % blob
+                        + "ALLOCATION_POOL=\"%s\"\n" % (allocation_pool, ))