Lib specific hacking rules

Starting with the most basic of the neutron rules. The neutron file will
include this one along with its own rules.

Partially-Implements: blueprint neutron-lib
Change-Id: Icda0c959c2ddbd1feada33e326f6c0fb0b23fc6a

HACKING.rst

@@ -1,4 +1,17 @@
 neutron-lib Style Commandments
 ===============================================
 
-Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/
+- Step 1: Read the OpenStack Style Commandments
+  http://docs.openstack.org/developer/hacking/
+- Step 2: Read on
+
+Neutron Specific Commandments
+-----------------------------
+
+- [N521] Validate that jsonutils module is used instead of json
+- [N523] Enforce namespace-less imports for oslo libraries
+- [N524] Prevent use of deprecated contextlib.nested.
+- [N525] Python 3: Do not use xrange.
+- [N526] Python 3: do not use basestring.
+- [N527] Python 3: do not use dict.iteritems.
+- [N529] Method's default argument shouldn't be mutable

neutron_lib/hacking/checks.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2014 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import re
+
+# TODO(dougwig)
+# add a rule for not importing neutron directly; most will have to ignore
+
+# Guidelines for writing new hacking checks
+#
+#  - Use only for Neutron specific tests. OpenStack general tests
+#    should be submitted to the common 'hacking' module.
+#  - Pick numbers in the range N5xx. Find the current test with
+#    the highest allocated number and then pick the next value.
+#  - Keep the test method code in the source file ordered based
+#    on the N5xx value.
+#  - List the new rule in the top level HACKING.rst file
+#  - Add test cases for each new rule to
+#    neutron_lib/tests/unit/hacking/test_checks.py
+
+mutable_default_args = re.compile(r"^\s*def .+\((.+=\{\}|.+=\[\])")
+
+oslo_namespace_imports_dot = re.compile(r"import[\s]+oslo[.][^\s]+")
+oslo_namespace_imports_from_dot = re.compile(r"from[\s]+oslo[.]")
+oslo_namespace_imports_from_root = re.compile(r"from[\s]+oslo[\s]+import[\s]+")
+contextlib_nested = re.compile(r"^with (contextlib\.)?nested\(")
+
+
+def use_jsonutils(logical_line, filename):
+    msg = "N521: jsonutils.%(fun)s must be used instead of json.%(fun)s"
+
+    # Some files in the tree are not meant to be run from inside Neutron
+    # itself, so we should not complain about them not using jsonutils
+    json_check_skipped_patterns = [
+        "neutron/plugins/ml2/drivers/openvswitch/agent/xenapi/etc/xapi.d/"
+        "plugins/netwrap",
+    ]
+
+    for pattern in json_check_skipped_patterns:
+        if pattern in filename:
+            return
+
+    if "json." in logical_line:
+        json_funcs = ['dumps(', 'dump(', 'loads(', 'load(']
+        for f in json_funcs:
+            pos = logical_line.find('json.%s' % f)
+            if pos != -1:
+                yield (pos, msg % {'fun': f[:-1]})
+
+
+def check_oslo_namespace_imports(logical_line):
+    if re.match(oslo_namespace_imports_from_dot, logical_line):
+        msg = ("N523: '%s' must be used instead of '%s'.") % (
+            logical_line.replace('oslo.', 'oslo_'),
+            logical_line)
+        yield(0, msg)
+    elif re.match(oslo_namespace_imports_from_root, logical_line):
+        msg = ("N523: '%s' must be used instead of '%s'.") % (
+            logical_line.replace('from oslo import ', 'import oslo_'),
+            logical_line)
+        yield(0, msg)
+    elif re.match(oslo_namespace_imports_dot, logical_line):
+        msg = ("N523: '%s' must be used instead of '%s'.") % (
+            logical_line.replace('import', 'from').replace('.', ' import '),
+            logical_line)
+        yield(0, msg)
+
+
+def check_no_contextlib_nested(logical_line, filename):
+    msg = ("N524: contextlib.nested is deprecated. With Python 2.7 and later "
+           "the with-statement supports multiple nested objects. See https://"
+           "docs.python.org/2/library/contextlib.html#contextlib.nested for "
+           "more information.")
+
+    if contextlib_nested.match(logical_line):
+        yield(0, msg)
+
+
+def check_python3_xrange(logical_line):
+    if re.search(r"\bxrange\s*\(", logical_line):
+        yield(0, "N525: Do not use xrange. Use range, or six.moves.range for "
+                 "large loops.")
+
+
+def check_no_basestring(logical_line):
+    if re.search(r"\bbasestring\b", logical_line):
+        msg = ("N526: basestring is not Python3-compatible, use "
+               "six.string_types instead.")
+        yield(0, msg)
+
+
+def check_python3_no_iteritems(logical_line):
+    if re.search(r".*\.iteritems\(\)", logical_line):
+        msg = ("N527: Use six.iteritems() instead of dict.iteritems().")
+        yield(0, msg)
+
+
+def no_mutable_default_args(logical_line):
+    msg = "N529: Method's default argument shouldn't be mutable!"
+    if mutable_default_args.match(logical_line):
+        yield (0, msg)
+
+
+def factory(register):
+    register(use_jsonutils)
+    register(check_oslo_namespace_imports)
+    register(check_no_contextlib_nested)
+    register(check_python3_xrange)
+    register(check_no_basestring)
+    register(check_python3_no_iteritems)
+    register(no_mutable_default_args)

neutron_lib/tests/unit/hacking/test_checks.py

@@ -0,0 +1,90 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import testtools
+
+from neutron_lib.hacking import checks
+from neutron_lib.tests import _base as base
+
+
+class HackingTestCase(base.BaseTestCase):
+
+    def assertLinePasses(self, func, line):
+        with testtools.ExpectedException(StopIteration):
+            next(func(line))
+
+    def assertLineFails(self, func, line):
+        self.assertIsInstance(next(func(line)), tuple)
+
+    def test_use_jsonutils(self):
+        def __get_msg(fun):
+            msg = ("N521: jsonutils.%(fun)s must be used instead of "
+                   "json.%(fun)s" % {'fun': fun})
+            return [(0, msg)]
+
+        for method in ('dump', 'dumps', 'load', 'loads'):
+            self.assertEqual(
+                __get_msg(method),
+                list(checks.use_jsonutils("json.%s(" % method,
+                                          "./neutron/common/rpc.py")))
+
+            self.assertEqual(0,
+                len(list(checks.use_jsonutils("jsonx.%s(" % method,
+                                              "./neutron/common/rpc.py"))))
+
+            self.assertEqual(0,
+                len(list(checks.use_jsonutils("json.%sx(" % method,
+                                              "./neutron/common/rpc.py"))))
+
+            self.assertEqual(0,
+                len(list(checks.use_jsonutils(
+                    "json.%s" % method,
+                    "./neutron/plugins/ml2/drivers/openvswitch/agent/xenapi/"
+                    "etc/xapi.d/plugins/netwrap"))))
+
+    def test_check_oslo_namespace_imports(self):
+        f = checks.check_oslo_namespace_imports
+        self.assertLinePasses(f, 'from oslo_utils import importutils')
+        self.assertLinePasses(f, 'import oslo_messaging')
+        self.assertLineFails(f, 'from oslo.utils import importutils')
+        self.assertLineFails(f, 'from oslo import messaging')
+        self.assertLineFails(f, 'import oslo.messaging')
+
+    def test_check_python3_xrange(self):
+        f = checks.check_python3_xrange
+        self.assertLineFails(f, 'a = xrange(1000)')
+        self.assertLineFails(f, 'b =xrange   (   42 )')
+        self.assertLineFails(f, 'c = xrange(1, 10, 2)')
+        self.assertLinePasses(f, 'd = range(1000)')
+        self.assertLinePasses(f, 'e = six.moves.range(1337)')
+
+    def test_no_basestring(self):
+        self.assertEqual(1,
+            len(list(checks.check_no_basestring("isinstance(x, basestring)"))))
+
+    def test_check_python3_iteritems(self):
+        f = checks.check_python3_no_iteritems
+        self.assertLineFails(f, "d.iteritems()")
+        self.assertLinePasses(f, "six.iteritems(d)")
+
+    def test_no_mutable_default_args(self):
+        self.assertEqual(1, len(list(checks.no_mutable_default_args(
+            " def fake_suds_context(calls={}):"))))
+
+        self.assertEqual(1, len(list(checks.no_mutable_default_args(
+            "def get_info_from_bdm(virt_type, bdm, mapping=[])"))))
+
+        self.assertEqual(0, len(list(checks.no_mutable_default_args(
+            "defined = []"))))
+
+        self.assertEqual(0, len(list(checks.no_mutable_default_args(
+            "defined, undefined = [], {}"))))

tox.ini

@@ -33,6 +33,8 @@ commands = sphinx-build -W -b html doc/source doc/build/html
 commands = oslo_debug_helper {posargs}
 
 [flake8]
+# E128 continuation line under-indented for visual indent
+ignore = E128
 show-source = True
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build