From 4adbecad993381cf74d23062edb691a0b148ddcd Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Sat, 21 Oct 2017 18:00:13 +0200 Subject: [PATCH] Fix flake8 fix a "bug" in that the flake8 configuration in tox.ini was exclusively selecting H231 as the only error it would report, so it was missing the errors in the python modules (such as submit_log_processor_jobs). Due to this being the case for a long time (since 2004) limit the more thorough linting to the roles/ and playbooks/ directories where we'll be adding ansible plugins/modules/etc. Also, lint in jenkins/script and nodepool. Fix problems found. We can lint everything with pep8 once the zuul v2 scripts are removed, not worth patching them right now. Change-Id: I479f010643cf3b67c183d763510f07a33400d38b Co-Authored-By: Jesse Keating --- jenkins/scripts/check_osc_commands.py | 4 ++-- jenkins/scripts/get-modulename.py | 1 - jenkins/scripts/project-requirements-change.py | 11 +++++------ jenkins/scripts/tardiff.py | 4 ++-- jenkins/scripts/xstatic_check_version.py | 2 +- .../extra-data.d/55-cache-devstack-repos | 10 +++++----- .../static/usr/local/bin/initialize-urandom.py | 11 +++++++---- .../openstack-repos/extra-data.d/50-create-repo-list | 11 +++++++---- .../library/submit_log_processor_jobs.py | 6 ++---- tools/check_gerrit_projects_changed.py | 8 ++++---- tools/check_valid_gerrit_projects.py | 2 +- tools/irc_tests.py | 8 ++++---- tools/normalize_acl.py | 3 +-- tools/normalize_channels_yaml.py | 7 ++++--- tools/normalize_projects_yaml.py | 1 + tox.ini | 5 +++-- 16 files changed, 49 insertions(+), 45 deletions(-) diff --git a/jenkins/scripts/check_osc_commands.py b/jenkins/scripts/check_osc_commands.py index ac2cb7a4e4..7daf5c8695 100755 --- a/jenkins/scripts/check_osc_commands.py +++ b/jenkins/scripts/check_osc_commands.py @@ -76,7 +76,7 @@ def find_duplicates(): # example: quota_show = openstackclient.common.quota:ShowQuota # and plugin entrypoints will not, for # example: orchestration = heatclient.osc.plugin - if not ':' in str(ep): + if ':' not in str(ep): continue # cliff does a mapping between spaces and underscores @@ -109,7 +109,7 @@ def find_duplicates(): print(overlap_cmds) # FIXME(stevemar): when we determine why commands are overlapping # we can uncomment the line below. - #return True + # return True # Safely return False here with the full set of commands print("Final set of commands...") diff --git a/jenkins/scripts/get-modulename.py b/jenkins/scripts/get-modulename.py index dc7cc602a8..e56fb38870 100755 --- a/jenkins/scripts/get-modulename.py +++ b/jenkins/scripts/get-modulename.py @@ -16,7 +16,6 @@ from __future__ import print_function import argparse import ConfigParser as configparser -import os import sys diff --git a/jenkins/scripts/project-requirements-change.py b/jenkins/scripts/project-requirements-change.py index 1d8c395d57..fb5ee431a9 100755 --- a/jenkins/scripts/project-requirements-change.py +++ b/jenkins/scripts/project-requirements-change.py @@ -138,10 +138,10 @@ def _is_requirement_in_global_reqs(req, global_reqs): # example: oslo.db[xyz]<1.2.3 for req2 in global_reqs: if (req.package == req2.package and - req.location == req2.location and - req.specifiers == req2.specifiers and - req.markers == req2.markers and - req.comment == req2.comment): + req.location == req2.location and + req.specifiers == req2.specifiers and + req.markers == req2.markers and + req.comment == req2.comment): return True return False @@ -220,7 +220,7 @@ def main(): for name, reqs in freqs.items(): counts = {} if (name in branch_reqs.reqs and - reqs == branch_reqs.reqs[name]): + reqs == branch_reqs.reqs[name]): # Unchanged [or a change that preserves a current value] continue if name in blacklist: @@ -257,7 +257,6 @@ def main(): ('[%s]' % extra) if extra else '', len(global_reqs[name]))) - # report the results if failed or head_reqs.failed or branch_reqs.failed: print("*** Incompatible requirement found!") diff --git a/jenkins/scripts/tardiff.py b/jenkins/scripts/tardiff.py index 8868e46997..cc90c4a5bf 100755 --- a/jenkins/scripts/tardiff.py +++ b/jenkins/scripts/tardiff.py @@ -96,7 +96,7 @@ class OpenStackTarDiff: if file not in self.sdist_files: self.missing_files.append(file) else: - #self.debug("file %s matches" % file) + # self.debug("file %s matches" % file) pass if len(self.missing_files) > 0: self.error("files missing in package: %s" % self.missing_files) @@ -167,7 +167,7 @@ class OpenStackTarDiff: (status, out) = commands.getstatusoutput(tar_cmd) if status != 0: self.error("command '%s' failed" % tar_cmd) - #self.debug(out) + # self.debug(out) self.sdist_files = out.split('\n') except Exception as err: self.error("command '%s' failed: %s" % (tar_cmd, err)) diff --git a/jenkins/scripts/xstatic_check_version.py b/jenkins/scripts/xstatic_check_version.py index 4112576aed..363f25e620 100755 --- a/jenkins/scripts/xstatic_check_version.py +++ b/jenkins/scripts/xstatic_check_version.py @@ -35,4 +35,4 @@ if xs is None: git_version = get_version() if git_version != xs.PACKAGE_VERSION: sys.exit('git tag version ({}) does not match package version ({})'. - format(git_version, xs.PACKAGE_VERSION)) + format(git_version, xs.PACKAGE_VERSION)) diff --git a/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos b/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos index 1f85a1888f..b8c5adcce0 100755 --- a/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos +++ b/nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos @@ -29,9 +29,9 @@ TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH'] # this is a bit weird; we want to get the filter to use on the # devstack lib/[apt|rpm] files to pre-install them. So we have to # swizzle things around -RELEASE=None +RELEASE = None if 'DIB_RELEASE' in os.environ: - RELEASE=os.environ['DIB_RELEASE'] + RELEASE = os.environ['DIB_RELEASE'] try: if open(os.path.join( TMP_MOUNT_PATH, @@ -39,20 +39,20 @@ try: # not actually used in devstack, but for consistency and because this # script will error if run on a platform where RELEASE isn't detected # as something - RELEASE='rhel6' + RELEASE = 'rhel6' except IOError: pass if 'DISTRO_NAME' in os.environ: if os.environ['DISTRO_NAME'] == 'centos7': # centos7 matches as rhel7 in devstack - RELEASE='rhel7' + RELEASE = 'rhel7' if not RELEASE: print("Can not determine RELEASE") sys.exit(1) DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack') CACHEDIR = os.path.join(TMP_MOUNT_PATH, 'tmp') -IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images') +IMAGES = os.path.join(TMP_HOOKS_PATH, 'source-repository-images') def run_local(cmd, status=False, cwd='.', env={}): diff --git a/nodepool/elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py b/nodepool/elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py index 547175a659..1cb449dcf8 100755 --- a/nodepool/elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py +++ b/nodepool/elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py @@ -153,9 +153,12 @@ short-lived as the nonblocking generator is being replaced[4] in current kernel versions and should not suffer from the same problem. [1] http://lxr.free-electrons.com/source/drivers/char/random.c?v=3.17#L1385 -[2] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=c6e9d6f38894798696f23c8084ca7edbf16ee895 -[3] https://www.freedesktop.org/software/systemd/man/systemd-random-seed.service.html -[4] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=e192be9d9a30555aae2ca1dc3aad37cba484cd4a +[2] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\ +?id=c6e9d6f38894798696f23c8084ca7edbf16ee895 +[3] https://www.freedesktop.org/software/systemd/man/systemd-random-seed.\ +service.html +[4] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\ +?id=e192be9d9a30555aae2ca1dc3aad37cba484cd4a """ @@ -180,7 +183,7 @@ class Pump(object): # Flags for getrandom: GRND_NONBLOCK = 0x0001 # Do not block - GRND_RANDOM = 0x0002 # Use /dev/random instead of urandom + GRND_RANDOM = 0x0002 # Use /dev/random instead of urandom def __init__(self): # Use ctypes to invoke getrandom since it is not available in diff --git a/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list b/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list index 31bf291abb..2157d10e93 100755 --- a/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list +++ b/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list @@ -27,11 +27,13 @@ from six.moves.urllib.error import URLError URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/' 'plain/gerrit/projects.yaml') -TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH'] -PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml') -GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org') +TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH'] +PROJECTS_REPOS = os.path.join(TMP_HOOKS_PATH, + 'source-repository-projects-yaml') +GIT_BASE = os.environ.get('GIT_BASE', 'git://git.openstack.org') + +CUSTOM_PROJECTS_LIST_URL = os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL') -CUSTOM_PROJECTS_LIST_URL=os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL') def get_project_list(url): try: @@ -56,6 +58,7 @@ def get_project_list(url): print("Could not open project list url: '%s'" % url) raise + def main(): projects = [] if CUSTOM_PROJECTS_LIST_URL: diff --git a/roles/submit-log-processor-jobs/library/submit_log_processor_jobs.py b/roles/submit-log-processor-jobs/library/submit_log_processor_jobs.py index e6e4521710..a782f170ee 100644 --- a/roles/submit-log-processor-jobs/library/submit_log_processor_jobs.py +++ b/roles/submit-log-processor-jobs/library/submit_log_processor_jobs.py @@ -18,13 +18,11 @@ import os import json import re -import logging from ansible.module_utils.six.moves import urllib from ansible.module_utils.basic import AnsibleModule, get_exception import gear -import yaml class FileMatcher(object): @@ -65,7 +63,7 @@ class LogMatcher(object): for (dirpath, dirnames, filenames) in os.walk(path): for filename in filenames: fn = os.path.join(dirpath, filename) - partial_name = fn[len(path)+1:] + partial_name = fn[len(path) + 1:] for matcher in self.matchers: if matcher.matches(partial_name): results.append(File(partial_name, matcher.tags)) @@ -140,7 +138,7 @@ def main(): argument_spec=dict( gearman_server=dict(type='str'), gearman_port=dict(type='int', default=4730), - #TODO: add ssl support + # TODO: add ssl support host_vars=dict(type='dict'), path=dict(type='path'), config=dict(type='dict'), diff --git a/tools/check_gerrit_projects_changed.py b/tools/check_gerrit_projects_changed.py index 8380766944..c18ceed143 100644 --- a/tools/check_gerrit_projects_changed.py +++ b/tools/check_gerrit_projects_changed.py @@ -17,7 +17,6 @@ import argparse import contextlib import git -import os import shutil import sys import tempfile @@ -39,13 +38,14 @@ def check_repo(repo_path): print("Checking git repo '%s':" % repo_path) try: with tempdir() as repopath: - repo = git.Repo.clone_from(repo_path, repopath) + git.Repo.clone_from(repo_path, repopath) print("Can be cloned") except Exception as e: print("Failure: %s" % e) found_errors += 1 return found_errors + def main(): found_errors = 0 @@ -67,7 +67,7 @@ def main(): projects_old = yaml.load(open(args.oldfile, 'r')) projects_new = yaml.load(open(args.newfile, 'r')) - ps_old={} + ps_old = {} for p in projects_old: name = p.get('project') ps_old[name] = p @@ -81,7 +81,7 @@ def main(): else: upstream_old = "" if (upstream != upstream_old and - 'track-upstream' in p.get('options', [])): + 'track-upstream' in p.get('options', [])): print("%s has changed" % name) found_errors += check_repo(upstream) diff --git a/tools/check_valid_gerrit_projects.py b/tools/check_valid_gerrit_projects.py index 4d188fb8da..01631503df 100755 --- a/tools/check_valid_gerrit_projects.py +++ b/tools/check_valid_gerrit_projects.py @@ -178,7 +178,7 @@ def main(): (entry, name)) # Check for valid options for option in p.get('options', []): - if not option in VALID_OPTIONS: + if option not in VALID_OPTIONS: found_errors += 1 print("ERROR: Unknown option '%s' in project %s" % (option, name)) diff --git a/tools/irc_tests.py b/tools/irc_tests.py index f5402de25c..e2c3a804a6 100755 --- a/tools/irc_tests.py +++ b/tools/irc_tests.py @@ -34,13 +34,13 @@ def access_gerrit_check(): gerrit_config = yaml.load(open('gerritbot/channels.yaml')) print("Basic check of gerritbot/channels.yaml") - REQUIRED_ENTRIES=("branches", "events", "projects") - VALID_EVENTS=("change-merged", "patchset-created", "x-vrif-minus-2") + REQUIRED_ENTRIES = ("branches", "events", "projects") + VALID_EVENTS = ("change-merged", "patchset-created", "x-vrif-minus-2") for channel in gerrit_config: for entry in REQUIRED_ENTRIES: if entry not in gerrit_config[channel]: - print("ERROR: Required entry '%s' not specified for channel '%s'" - % (entry, channel)) + print("ERROR: Required entry '%s' not specified for channel " + "'%s'" % (entry, channel)) errors = True elif not gerrit_config[channel][entry]: print("ERROR: Entry '%s' has no content for channel '%s'" diff --git a/tools/normalize_acl.py b/tools/normalize_acl.py index a771b31798..bcf49ed16e 100755 --- a/tools/normalize_acl.py +++ b/tools/normalize_acl.py @@ -76,8 +76,7 @@ valid_keys = {'abandon', 'requireChangeId', 'requireContributorAgreement', 'state', - 'value' -} + 'value'} if '0' in transformations or not transformations: dry_run = True diff --git a/tools/normalize_channels_yaml.py b/tools/normalize_channels_yaml.py index 5d5ad29b0e..df66175624 100755 --- a/tools/normalize_channels_yaml.py +++ b/tools/normalize_channels_yaml.py @@ -16,12 +16,12 @@ # limitations under the License. import locale -import os import sys import yaml from collections import OrderedDict import projectconfig_yamllib as pcy + def main(): locale.setlocale(locale.LC_COLLATE, 'C') @@ -32,7 +32,7 @@ def main(): Dumper=pcy.IndentedDumper) chandata = yaml.load(open('gerritbot/channels.yaml')) - for k,v in chandata.items(): + for k, v in chandata.items(): v['projects'] = sorted(v['projects']) sys.stdout.write('# This file is sorted alphabetically by channel name.\n') @@ -42,7 +42,8 @@ def main(): sys.stdout.write('\n') first = False sys.stdout.write(yaml.dump({k: chandata[k]}, default_flow_style=False, - Dumper=pcy.IndentedDumper, width=80, indent=2)) + Dumper=pcy.IndentedDumper, width=80, + indent=2)) if __name__ == '__main__': main() diff --git a/tools/normalize_projects_yaml.py b/tools/normalize_projects_yaml.py index 2de37ec3e3..f8635fb640 100755 --- a/tools/normalize_projects_yaml.py +++ b/tools/normalize_projects_yaml.py @@ -19,6 +19,7 @@ import yaml from collections import OrderedDict import projectconfig_yamllib as pcy + def main(): yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, pcy.construct_yaml_map) diff --git a/tox.ini b/tox.ini index e7d682d42f..ac5cfc8a56 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,9 @@ passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY [testenv:linters] commands = {toxinidir}/tools/run-bashate.sh - flake8 + # TODO(jaegerandi): Remove parameter to test everywhere once + # Zuul v2 files have been removed + flake8 jenkins/scripts roles playbooks nodepool [testenv:gerrit] deps = PyYAML @@ -39,7 +41,6 @@ commands = show-source = True exclude = .tox,.test ignore = E125,H -select = H231 [testenv:grafyaml] basepython = python2.7