Add test cases for tox line comment parsing

Tox line comment parsing had a few side effects at first so add
testing to it.

Change-Id: I9eb2aa00d6e467bd170fad80598728aec6949ee6

roles/tox/library/test-cases/no_output.yaml

@@ -0,0 +1,5 @@
+---
+workdir: .
+output: ""
+
+comments: {}

roles/tox/library/test-cases/non_existing_files.yaml

@@ -0,0 +1,7 @@
+---
+workdir: .
+output: |
+  {{1}} keystoneauth1.tests.unit.test_fair_sempahore.SemaphoreTests.test_semaphore_fast_multiple_concurrency [0.004427s] ... ok
+  stdin:1:1: K333  'import oslo_utils' must be used instead of 'import oslo.utils'.
+  /dev/stdin:3:1: K333  'import oslo_utils.encodeutils' must be used instead of 'import oslo.utils.encodeutils'.
+comments: {}

roles/tox/library/test-cases/pep8_output.yaml

@@ -0,0 +1,26 @@
+---
+workdir: .
+output: |
+  tests/__init__.py:86:12: E111 indentation is not a multiple of four
+             return self._normalize_machine(self.baremetal.get_node(name_or_id))
+             ^
+  ./tests/__init__.py:90:32: E231 missing whitespace after ','
+      def get_machine_by_mac(self,mac):
+                                 ^
+  # This should be ignored
+  /openstack/cloud/_baremetal.py:218:46: E128 continuation line under-indented for visual indent
+                                              json=kwargs,
+                                              ^
+  tests/test_discovery.py:219:46: E128 continuation line under-indented for visual indent
+                                               error_message=msg,
+                                               ^
+comments:
+  tests/__init__.py:
+    - line: 86
+      message: E111 indentation is not a multiple of four
+    - line: 90
+      message: E231 missing whitespace after ','
+  tests/test_discovery.py:
+    - line: 219
+      message: E128 continuation line under-indented for visual indent
+

roles/tox/library/test-cases/pep8_output_envlist.yaml

@@ -0,0 +1,27 @@
+---
+workdir: .
+tox_envlist: pep8
+output: |
+  tests/__init__.py:86:12: E111 indentation is not a multiple of four
+             return self._normalize_machine(self.baremetal.get_node(name_or_id))
+             ^
+  ./tests/__init__.py:90:32: E231 missing whitespace after ','
+      def get_machine_by_mac(self,mac):
+                                 ^
+  # This should be ignored
+  /openstack/cloud/_baremetal.py:218:46: E128 continuation line under-indented for visual indent
+                                              json=kwargs,
+                                              ^
+  tests/test_discovery.py:219:46: E128 continuation line under-indented for visual indent
+                                               error_message=msg,
+                                               ^
+comments:
+  tests/__init__.py:
+    - line: 86
+      message: "pep8: E111 indentation is not a multiple of four"
+    - line: 90
+      message: "pep8: E231 missing whitespace after ','"
+  tests/test_discovery.py:
+    - line: 219
+      message: "pep8: E128 continuation line under-indented for visual indent"
+

roles/tox/library/test-cases/pylint_output_colored.yaml

@@ -0,0 +1,21 @@
+---
+workdir: .
+output: |
+  tests/__init__.py:2808:8: E1120: No value for argument 'get_storage_profile_id' in method call (no-value-for-parameter)
+  tests/__init__.py:2808:8: E1120: No value for argument 'select_ds_for_volume' in method call (no-value-for-parameter)
+  tests/__init__.py:2812:8: E1120: No value for argument 'get_adapter_type' in method call (no-value-for-parameter)
+  tests/__init__.py:2812:8: E1120: No value for argument 'get_disk_type' in method call (no-value-for-parameter)
+  tests/__init__.py:2812:8: E1120: No value for argument 'vops' in method call (no-value-for-parameter)
+comments:
+  tests/__init__.py:
+    - line: 2808
+      message: "E1120: No value for argument 'get_storage_profile_id' in method call (no-value-for-parameter)"
+    - line: 2808
+      message: "E1120: No value for argument 'select_ds_for_volume' in method call (no-value-for-parameter)"
+    - line: 2812
+      message: "E1120: No value for argument 'get_adapter_type' in method call (no-value-for-parameter)"
+    - line: 2812
+      message: "E1120: No value for argument 'get_disk_type' in method call (no-value-for-parameter)"
+    - line: 2812
+      message: "E1120: No value for argument 'vops' in method call (no-value-for-parameter)"
+

roles/tox/library/test-cases/sphinx_output.yaml

@@ -0,0 +1,24 @@
+---
+workdir: .
+output: |
+  reading sources... [ 95%] reference/web
+  reading sources... [ 97%] tutorials/admin
+  reading sources... [ 98%] tutorials/quick-start
+  reading sources... [100%] tutorials/user
+
+  unable to find release notes file associated with unique id '4b4ecb4678c11470', skipping
+  unable to find release notes file associated with unique id 'df1645c63e82f65c', skipping
+
+  Warning, treated as error:
+  {workdir}/CONTRIBUTING.rst:96:Unknown directive type "code-blockf".
+
+  .. code-blockf:: shell
+
+     echo "test" > README.txt
+     git add .
+     git commit -m "First commit"
+comments:
+  CONTRIBUTING.rst:
+    - line: 96
+      message: Unknown directive type "code-blockf".
+

roles/tox/library/test_tox_parse_output.py

@@ -0,0 +1,33 @@
+# Copyright (C) 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import testtools
+
+from tests import generate_dynamic_comments_tests
+from .tox_parse_output import extract_file_comments
+
+TESTS_DIR = os.path.join(os.path.dirname(__file__),
+                         'test-cases')
+
+
+class TestToxParseOutput(testtools.TestCase):
+    pass
+
+
+generate_dynamic_comments_tests(TestToxParseOutput, TESTS_DIR,
+                                extract_file_comments)

roles/tox/library/tox_parse_output.py

@@ -89,7 +89,8 @@ def extract_line_comment(line):
     return file_path, start_line, message
 
 
-def extract_file_comments(tox_output, tox_envlist):
+def extract_file_comments(tox_output, workdir, tox_envlist=None):
+    os.chdir(workdir)
     ret = {}
     for line in tox_output.split('\n'):
         if not line:
@@ -139,9 +140,9 @@ def main():
     )
     tox_output = module.params['tox_output']
     tox_envlist = module.params['tox_envlist']
-    os.chdir(module.params['workdir'])
 
-    file_comments = extract_file_comments(tox_output, tox_envlist)
+    file_comments = extract_file_comments(
+        tox_output, module.params['workdir'], tox_envlist)
     module.exit_json(changed=False, file_comments=file_comments)
 
 

tests/__init__.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import os
-
+import re
 import yaml
 
 
@@ -24,8 +24,18 @@ def generate_dynamic_comments_tests(cls, test_path, func):
         def test(self):
             path = "%s/%s" % (test_path, name)
             with open(path) as fd:
+                # Don't filter unicode chars as we need to parse \x1B
+                yaml.reader.Reader.NON_PRINTABLE = re.compile('.^')
                 data = yaml.load(fd, Loader=yaml.FullLoader)
-            comments = func(data['output'], data['workdir'])
+            extra_args = {
+                arg: data[arg] for arg in data
+                if arg not in ('output', 'comments')
+            }
+
+            # Replace workdir in output by current work dir
+            data['output'] = data['output'].format(workdir=os.getcwd())
+
+            comments = func(data['output'], **extra_args)
             self.assertEqual(data['comments'], comments)
         return test