From 172d34813dfbcbbba68552f7f607483c8f5eca5f Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Wed, 10 Apr 2019 15:28:33 +0100 Subject: [PATCH] Bump flake8 version to something modern The flake8 version we're using is a good 2 years old and is in turn using an old version of pycodestyle. This is causing the following warnings: FutureWarning: Possible nested set at position 1 EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') Bump to the latest and greatest so we get these fixes. This involves dropping support for local-checks, which don't seem to work in this version. Fortunately, in the intervening time, flake8 has grown the ability to do local checks all by its lonesome. The tests also need to be reworked and unfortunately are much slower now. This is because flake8 3.x's API is file-based [1] and without rooting around the guts of flake8, it's not practical to do things any other way. Hopefully flake8 4.x won't have such issues. [1] https://gitlab.com/pycqa/flake8/issues/545 Change-Id: I695ff02a6970663add10caf7f16a66abf9d1239d Signed-off-by: Stephen Finucane Sem-Ver: api-break --- doc/requirements.txt | 3 +- hacking/core.py | 27 -------- hacking/tests/test_doctest.py | 68 ++++++++----------- hacking/tests/test_local.py | 36 ---------- lower-constraints.txt | 13 ++-- .../flake8-3-x-support-cd478de79fe7b63d.yaml | 12 ++++ requirements.txt | 7 +- setup.cfg | 1 - 8 files changed, 49 insertions(+), 118 deletions(-) delete mode 100644 hacking/tests/test_local.py create mode 100644 releasenotes/notes/flake8-3-x-support-cd478de79fe7b63d.yaml diff --git a/doc/requirements.txt b/doc/requirements.txt index b9c5e1f1..1af3f20d 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,4 @@ -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinx>=1.8.0,<2.0.0;python_version=='2.7' # BSD +sphinx>=1.8.0,!=2.1.0;python_version>='3.4' # BSD openstackdocstheme>=1.18.1 # Apache-2.0 reno>=2.5.0 # Apache-2.0 diff --git a/hacking/core.py b/hacking/core.py index c96f71e1..89fb2b8b 100644 --- a/hacking/core.py +++ b/hacking/core.py @@ -21,9 +21,6 @@ Built as a sets of pycodestyle checks using flake8. import gettext import sys -import pbr.util -import pycodestyle - from hacking import config # Import tests need to inject _ properly into the builtins @@ -138,27 +135,3 @@ class GlobalCheck(object): def run_once(self): pass - - -class ProxyChecks(GlobalCheck): - """Provide a mechanism for locally defined checks.""" - name = 'ProxyChecker' - - @classmethod - def add_options(cls, parser): - # We're looking for local checks, so we need to include the local - # dir in the search path - sys.path.append('.') - - local_check = CONF.get_multiple('local-check', default=[]) - for check_path in set(local_check): - if check_path.strip(): - checker = pbr.util.resolve_name(check_path) - pycodestyle.register_check(checker) - - local_check_fact = CONF.get('local-check-factory') - if local_check_fact: - factory = pbr.util.resolve_name(local_check_fact) - factory(pycodestyle.register_check) - - sys.path.pop() diff --git a/hacking/tests/test_doctest.py b/hacking/tests/test_doctest.py index 007e8bd3..e3e0a486 100644 --- a/hacking/tests/test_doctest.py +++ b/hacking/tests/test_doctest.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,22 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re - -from flake8 import engine -import pycodestyle +import subprocess +import sys +import tempfile import pkg_resources import six import testscenarios from testtools import content -from testtools import matchers -import hacking import hacking.tests SELFTEST_REGEX = re.compile(r'\b(Okay|[HEW]\d{3}):\s(.*)') -# Each scenario is (name, dict(lines=.., options=..., code=...)) +# Each scenario is (name, {lines=.., raw=..., code=...}) file_cases = [] @@ -37,32 +35,28 @@ class HackingTestCase(hacking.tests.TestCase): scenarios = file_cases - def test_pycodestyle(self): + def test_flake8(self): - # NOTE(jecarey): Add tests marked as off_by_default to enable testing - turn_on = set(['H106', 'H203', 'H904', 'H204', 'H205', 'H210']) - if self.options.select: - turn_on.update(self.options.select) - self.options.select = tuple(turn_on) + with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: + f.write(''.join(self.lines)) + + cmd = [sys.executable, '-mflake8', '--isolated', + '--select=%s' % self.code, '--ignore=F', + '--format=%(code)s\t%(path)s\t%(row)d', f.name] + out, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() + + out = out.decode('utf-8') - report = pycodestyle.BaseReport(self.options) - checker = pycodestyle.Checker(lines=self.lines, options=self.options, - report=report) - checker.check_all() - self.addDetail('doctest', content.text_content(self.raw)) if self.code == 'Okay': - self.assertThat( - len(report.counters), - matchers.Not(matchers.GreaterThan( - len(self.options.benchmark_keys))), - "incorrectly found %s" % ', '.join( - [key for key in report.counters - if key not in self.options.benchmark_keys])) + self.assertEqual('', out) else: self.addDetail('reason', content.text_content("Failed to trigger rule %s" % self.code)) - self.assertIn(self.code, report.counters) + self.assertNotEqual('', out) + self.assertEqual(self.code, out.split('\t')[0].rstrip(':'), out) + + os.remove(f.name) def _get_lines(check): @@ -71,28 +65,26 @@ def _get_lines(check): match = SELFTEST_REGEX.match(line) if match is None: continue - yield (line, match.groups()) + yield line, match.groups() def load_tests(loader, tests, pattern): - flake8_style = engine.get_style_guide(parse_argv=False, - # Ignore H104 otherwise it's - # raised on doctests. - ignore=('F', 'H104')) - options = flake8_style.options - for entry in pkg_resources.iter_entry_points('flake8.extension'): if not entry.module_name.startswith('hacking.'): continue + check = entry.load() - name = entry.attrs[0] if check.skip_on_py3 and six.PY3: continue - for (lineno, (raw, (code, source))) in enumerate(_get_lines(check)): + + name = entry.attrs[0] + for lineno, (raw, (code, source)) in enumerate(_get_lines(check)): lines = [part.replace(r'\t', '\t') + '\n' for part in source.split(r'\n')] - file_cases.append(("%s-%s-line-%s" % (entry.name, name, lineno), - dict(lines=lines, raw=raw, options=options, - code=code))) + file_cases.append(( + '%s-%s-line-%s' % (entry.name, name, lineno), + {'lines': lines, 'raw': raw, 'code': code}, + )) + return testscenarios.load_tests_apply_scenarios(loader, tests, pattern) diff --git a/hacking/tests/test_local.py b/hacking/tests/test_local.py deleted file mode 100644 index cba64410..00000000 --- a/hacking/tests/test_local.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flake8 import engine -import pycodestyle - -import hacking.tests - - -def check(physical_line): - """Test check to make sure local-checks are working.""" - if physical_line.strip() == "#this-is-the-test-phrase": - return (0, "L100: Found local-check test case") - - -class HackingTestCase(hacking.tests.TestCase): - def test_local_check(self): - flake8_style = engine.get_style_guide(parse_argv=False, ignore='F') - report = pycodestyle.BaseReport(flake8_style.options) - line = ["#this-is-the-test-phrase"] - checker = pycodestyle.Checker(lines=line, options=flake8_style.options, - report=report) - checker.check_all() - self.assertIn("L100", report.counters) diff --git a/lower-constraints.txt b/lower-constraints.txt index db18d950..600340bf 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -6,18 +6,15 @@ dulwich==0.15.0 eventlet==0.18.2 extras==1.0.0 fixtures==3.0.0 -flake8==2.6.2 -greenlet==0.4.10 +flake8==3.6.0 imagesize==0.7.1 Jinja2==2.10 linecache2==1.0.0 MarkupSafe==1.0 -mccabe==0.5.3 +mccabe==0.6.0 mock==2.0.0 -openstackdocstheme==1.18.1 -pbr==2.0.0 -pycodestyle==2.0.0 -pyflakes==1.2.3 +pycodestyle==2.4.0 +pyflakes==2.0.0 Pygments==2.2.0 python-mimeparse==1.6.0 python-subunit==1.0.0 @@ -27,8 +24,6 @@ reno==2.5.0 requests==2.14.2 six==1.10.0 snowballstemmer==1.2.1 -Sphinx==1.6.5 -sphinxcontrib-websupport==1.0.1 stestr==2.0.0 testscenarios==0.4 testtools==2.2.0 diff --git a/releasenotes/notes/flake8-3-x-support-cd478de79fe7b63d.yaml b/releasenotes/notes/flake8-3-x-support-cd478de79fe7b63d.yaml new file mode 100644 index 00000000..4f155427 --- /dev/null +++ b/releasenotes/notes/flake8-3-x-support-cd478de79fe7b63d.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + *hacking* is now compatible with flake8 3.x, which is a large rewrite of + flake8. flake8 2.x is no longer supported. +upgrade: + - | + Support for local checks has been removed. This was not compatible with + *flake8* 3.x. Users should migrate to the flake8's `native local plugins + support`__. + + __ https://flake8.pycqa.org/en/3.7.0/user/configuration.html#using-local-plugins diff --git a/requirements.txt b/requirements.txt index b1c52bb2..bd5cd439 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,2 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. -pbr # Apache-2.0 - -flake8<2.7.0,>=2.6.0 # MIT +flake8<4.0.0,>=3.6.0 # MIT six>=1.10.0 # MIT diff --git a/setup.cfg b/setup.cfg index c01720ec..4ed7b6b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,6 @@ packages = [entry_points] flake8.extension = - H000 = hacking.core:ProxyChecks H101 = hacking.checks.comments:hacking_todo_format H102 = hacking.checks.comments:hacking_has_license H103 = hacking.checks.comments:hacking_has_correct_license