Fix flake8

fix a "bug" in that the flake8 configuration in tox.ini was exclusively
selecting H231 as the only error it would report, so it was missing the
errors in the python modules (such as submit_log_processor_jobs). Due to
this being the case for a long time (since 2004) limit the more thorough
linting to the roles/ and playbooks/ directories where we'll be adding
ansible plugins/modules/etc. Also, lint in jenkins/script and nodepool.

Fix problems found.

We can lint everything with pep8 once the zuul v2 scripts are removed,
not worth patching them right now.

Change-Id: I479f010643cf3b67c183d763510f07a33400d38b
Co-Authored-By: Jesse Keating <omgjlk@us.ibm.com>

jenkins/scripts/check_osc_commands.py

@@ -76,7 +76,7 @@ def find_duplicates():
             # example: quota_show = openstackclient.common.quota:ShowQuota
             # and plugin entrypoints will not, for
             # example: orchestration = heatclient.osc.plugin
-            if not ':' in str(ep):
+            if ':' not in str(ep):
                 continue
 
             # cliff does a mapping between spaces and underscores
@@ -109,7 +109,7 @@ def find_duplicates():
         print(overlap_cmds)
         # FIXME(stevemar): when we determine why commands are overlapping
         # we can uncomment the line below.
-        #return True
+        # return True
 
     # Safely return False here with the full set of commands
     print("Final set of commands...")

jenkins/scripts/get-modulename.py

@@ -16,7 +16,6 @@ from __future__ import print_function
 
 import argparse
 import ConfigParser as configparser
-import os
 import sys
 
 

jenkins/scripts/project-requirements-change.py

@@ -138,10 +138,10 @@ def _is_requirement_in_global_reqs(req, global_reqs):
     # example: oslo.db[xyz]<1.2.3
     for req2 in global_reqs:
         if (req.package == req2.package and
-                    req.location == req2.location and
-                    req.specifiers == req2.specifiers and
-                    req.markers == req2.markers and
-                    req.comment == req2.comment):
+                req.location == req2.location and
+                req.specifiers == req2.specifiers and
+                req.markers == req2.markers and
+                req.comment == req2.comment):
             return True
     return False
 
@@ -220,7 +220,7 @@ def main():
             for name, reqs in freqs.items():
                 counts = {}
                 if (name in branch_reqs.reqs and
-                            reqs == branch_reqs.reqs[name]):
+                        reqs == branch_reqs.reqs[name]):
                     # Unchanged [or a change that preserves a current value]
                     continue
                 if name in blacklist:
@@ -257,7 +257,6 @@ def main():
                                   ('[%s]' % extra) if extra else '',
                                   len(global_reqs[name])))
 
-
     # report the results
     if failed or head_reqs.failed or branch_reqs.failed:
         print("*** Incompatible requirement found!")

jenkins/scripts/tardiff.py

@@ -96,7 +96,7 @@ class OpenStackTarDiff:
             if file not in self.sdist_files:
                 self.missing_files.append(file)
             else:
-                #self.debug("file %s matches" % file)
+                # self.debug("file %s matches" % file)
                 pass
         if len(self.missing_files) > 0:
             self.error("files missing in package: %s" % self.missing_files)
@@ -167,7 +167,7 @@ class OpenStackTarDiff:
             (status, out) = commands.getstatusoutput(tar_cmd)
             if status != 0:
                 self.error("command '%s' failed" % tar_cmd)
-            #self.debug(out)
+            # self.debug(out)
             self.sdist_files = out.split('\n')
         except Exception as err:
             self.error("command '%s' failed: %s" % (tar_cmd, err))

jenkins/scripts/xstatic_check_version.py

@@ -35,4 +35,4 @@ if xs is None:
 git_version = get_version()
 if git_version != xs.PACKAGE_VERSION:
     sys.exit('git tag version ({}) does not match package version ({})'.
-        format(git_version, xs.PACKAGE_VERSION))
+             format(git_version, xs.PACKAGE_VERSION))

nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos

@@ -29,9 +29,9 @@ TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH']
 # this is a bit weird; we want to get the filter to use on the
 # devstack lib/[apt|rpm] files to pre-install them.  So we have to
 # swizzle things around
-RELEASE=None
+RELEASE = None
 if 'DIB_RELEASE' in os.environ:
-    RELEASE=os.environ['DIB_RELEASE']
+    RELEASE = os.environ['DIB_RELEASE']
 try:
     if open(os.path.join(
             TMP_MOUNT_PATH,
@@ -39,20 +39,20 @@ try:
         # not actually used in devstack, but for consistency and because this
         # script will error if run on a platform where RELEASE isn't detected
         # as something
-        RELEASE='rhel6'
+        RELEASE = 'rhel6'
 except IOError:
     pass
 if 'DISTRO_NAME' in os.environ:
     if os.environ['DISTRO_NAME'] == 'centos7':
         # centos7 matches as rhel7 in devstack
-        RELEASE='rhel7'
+        RELEASE = 'rhel7'
 if not RELEASE:
     print("Can not determine RELEASE")
     sys.exit(1)
 
 DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack')
 CACHEDIR = os.path.join(TMP_MOUNT_PATH, 'tmp')
-IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images')
+IMAGES = os.path.join(TMP_HOOKS_PATH, 'source-repository-images')
 
 
 def run_local(cmd, status=False, cwd='.', env={}):

nodepool/elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py

@@ -153,9 +153,12 @@ short-lived as the nonblocking generator is being replaced[4] in
 current kernel versions and should not suffer from the same problem.
 
 [1] http://lxr.free-electrons.com/source/drivers/char/random.c?v=3.17#L1385
-[2] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=c6e9d6f38894798696f23c8084ca7edbf16ee895
-[3] https://www.freedesktop.org/software/systemd/man/systemd-random-seed.service.html
-[4] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=e192be9d9a30555aae2ca1dc3aad37cba484cd4a
+[2] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\
+?id=c6e9d6f38894798696f23c8084ca7edbf16ee895
+[3] https://www.freedesktop.org/software/systemd/man/systemd-random-seed.\
+service.html
+[4] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\
+?id=e192be9d9a30555aae2ca1dc3aad37cba484cd4a
 """
 
 
@@ -180,7 +183,7 @@ class Pump(object):
 
     # Flags for getrandom:
     GRND_NONBLOCK = 0x0001  # Do not block
-    GRND_RANDOM   = 0x0002  # Use /dev/random instead of urandom
+    GRND_RANDOM = 0x0002  # Use /dev/random instead of urandom
 
     def __init__(self):
         # Use ctypes to invoke getrandom since it is not available in

nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list

@@ -27,11 +27,13 @@ from six.moves.urllib.error import URLError
 URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
        'plain/gerrit/projects.yaml')
 
-TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH']
-PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml')
-GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org')
+TMP_HOOKS_PATH = os.environ['TMP_HOOKS_PATH']
+PROJECTS_REPOS = os.path.join(TMP_HOOKS_PATH,
+                              'source-repository-projects-yaml')
+GIT_BASE = os.environ.get('GIT_BASE', 'git://git.openstack.org')
+
+CUSTOM_PROJECTS_LIST_URL = os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL')
 
-CUSTOM_PROJECTS_LIST_URL=os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL')
 
 def get_project_list(url):
     try:
@@ -56,6 +58,7 @@ def get_project_list(url):
         print("Could not open project list url: '%s'" % url)
         raise
 
+
 def main():
     projects = []
     if CUSTOM_PROJECTS_LIST_URL:

roles/submit-log-processor-jobs/library/submit_log_processor_jobs.py

@@ -18,13 +18,11 @@
 import os
 import json
 import re
-import logging
 
 from ansible.module_utils.six.moves import urllib
 from ansible.module_utils.basic import AnsibleModule, get_exception
 
 import gear
-import yaml
 
 
 class FileMatcher(object):
@@ -65,7 +63,7 @@ class LogMatcher(object):
         for (dirpath, dirnames, filenames) in os.walk(path):
             for filename in filenames:
                 fn = os.path.join(dirpath, filename)
-                partial_name = fn[len(path)+1:]
+                partial_name = fn[len(path) + 1:]
                 for matcher in self.matchers:
                     if matcher.matches(partial_name):
                         results.append(File(partial_name, matcher.tags))
@@ -140,7 +138,7 @@ def main():
         argument_spec=dict(
             gearman_server=dict(type='str'),
             gearman_port=dict(type='int', default=4730),
-            #TODO: add ssl support
+            # TODO: add ssl support
             host_vars=dict(type='dict'),
             path=dict(type='path'),
             config=dict(type='dict'),

tools/check_gerrit_projects_changed.py

@@ -17,7 +17,6 @@
 import argparse
 import contextlib
 import git
-import os
 import shutil
 import sys
 import tempfile
@@ -39,13 +38,14 @@ def check_repo(repo_path):
     print("Checking git repo '%s':" % repo_path)
     try:
         with tempdir() as repopath:
-            repo = git.Repo.clone_from(repo_path, repopath)
+            git.Repo.clone_from(repo_path, repopath)
         print("Can be cloned")
     except Exception as e:
         print("Failure: %s" % e)
         found_errors += 1
     return found_errors
 
+
 def main():
     found_errors = 0
 
@@ -67,7 +67,7 @@ def main():
     projects_old = yaml.load(open(args.oldfile, 'r'))
     projects_new = yaml.load(open(args.newfile, 'r'))
 
-    ps_old={}
+    ps_old = {}
     for p in projects_old:
         name = p.get('project')
         ps_old[name] = p
@@ -81,7 +81,7 @@ def main():
         else:
             upstream_old = ""
         if (upstream != upstream_old and
-            'track-upstream' in p.get('options', [])):
+                'track-upstream' in p.get('options', [])):
             print("%s has changed" % name)
             found_errors += check_repo(upstream)
 

tools/check_valid_gerrit_projects.py

@@ -178,7 +178,7 @@ def main():
                       (entry, name))
         # Check for valid options
         for option in p.get('options', []):
-            if not option in VALID_OPTIONS:
+            if option not in VALID_OPTIONS:
                 found_errors += 1
                 print("ERROR: Unknown option '%s' in project %s" %
                       (option, name))

tools/irc_tests.py

@@ -34,13 +34,13 @@ def access_gerrit_check():
     gerrit_config = yaml.load(open('gerritbot/channels.yaml'))
 
     print("Basic check of gerritbot/channels.yaml")
-    REQUIRED_ENTRIES=("branches", "events", "projects")
-    VALID_EVENTS=("change-merged", "patchset-created", "x-vrif-minus-2")
+    REQUIRED_ENTRIES = ("branches", "events", "projects")
+    VALID_EVENTS = ("change-merged", "patchset-created", "x-vrif-minus-2")
     for channel in gerrit_config:
         for entry in REQUIRED_ENTRIES:
             if entry not in gerrit_config[channel]:
-                print("ERROR: Required entry '%s' not specified for channel '%s'"
-                      % (entry, channel))
+                print("ERROR: Required entry '%s' not specified for channel "
+                      "'%s'" % (entry, channel))
                 errors = True
             elif not gerrit_config[channel][entry]:
                 print("ERROR: Entry '%s' has no content for channel '%s'"

tools/normalize_acl.py

@@ -76,8 +76,7 @@ valid_keys = {'abandon',
               'requireChangeId',
               'requireContributorAgreement',
               'state',
-              'value'
-}
+              'value'}
 
 if '0' in transformations or not transformations:
     dry_run = True

tools/normalize_channels_yaml.py

@@ -16,12 +16,12 @@
 # limitations under the License.
 
 import locale
-import os
 import sys
 import yaml
 from collections import OrderedDict
 import projectconfig_yamllib as pcy
 
+
 def main():
     locale.setlocale(locale.LC_COLLATE, 'C')
 
@@ -32,7 +32,7 @@ def main():
                          Dumper=pcy.IndentedDumper)
 
     chandata = yaml.load(open('gerritbot/channels.yaml'))
-    for k,v in chandata.items():
+    for k, v in chandata.items():
         v['projects'] = sorted(v['projects'])
 
     sys.stdout.write('# This file is sorted alphabetically by channel name.\n')
@@ -42,7 +42,8 @@ def main():
             sys.stdout.write('\n')
         first = False
         sys.stdout.write(yaml.dump({k: chandata[k]}, default_flow_style=False,
-                            Dumper=pcy.IndentedDumper, width=80, indent=2))
+                                   Dumper=pcy.IndentedDumper, width=80,
+                                   indent=2))
 
 if __name__ == '__main__':
     main()

tools/normalize_projects_yaml.py

@@ -19,6 +19,7 @@ import yaml
 from collections import OrderedDict
 import projectconfig_yamllib as pcy
 
+
 def main():
     yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                          pcy.construct_yaml_map)

tox.ini

@@ -10,7 +10,9 @@ passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
 [testenv:linters]
 commands =
   {toxinidir}/tools/run-bashate.sh
-  flake8
+  # TODO(jaegerandi): Remove parameter to test everywhere once
+  # Zuul v2 files have been removed
+  flake8 jenkins/scripts roles playbooks nodepool
 
 [testenv:gerrit]
 deps = PyYAML
@@ -39,7 +41,6 @@ commands =
 show-source = True
 exclude = .tox,.test
 ignore = E125,H
-select = H231
 
 [testenv:grafyaml]
 basepython = python2.7