From b934c1a0528b2fdc2cb49e5c9ca1bb3d27f6c4f2 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 16 Jun 2017 19:31:47 -0500 Subject: [PATCH] Remove use of six library It exists only for py2/py3 compat. We do not need it any more. This will explicitly break Zuul v3 for python2, which is different than simply ceasing to test it and no longer declaring we support it. Since we're not testing it any longer, it's bound to degrade overtime without us noticing, so hopefully a clean and explicit break will prevent people from running under python2 and it working for a minute, then breaking later. Change-Id: Ia16bb399a2869ab37a183f3f2197275bb3acafee --- requirements.txt | 1 - tests/base.py | 49 +++++++++++++------------- tests/unit/test_scheduler.py | 2 +- tests/unit/test_webapp.py | 2 +- tools/encrypt_secret.py | 2 +- zuul/cmd/__init__.py | 8 ++--- zuul/configloader.py | 9 +++-- zuul/connection/__init__.py | 4 +-- zuul/driver/__init__.py | 20 ++++------- zuul/driver/bubblewrap/__init__.py | 5 ++- zuul/driver/gerrit/gerritconnection.py | 8 ++--- zuul/driver/git/gitconnection.py | 2 +- zuul/executor/server.py | 4 +-- zuul/lib/clonemapper.py | 8 ++--- zuul/lib/cloner.py | 4 +-- zuul/lib/commandsocket.py | 2 +- zuul/model.py | 6 +--- zuul/reporter/__init__.py | 5 +-- zuul/rpclistener.py | 4 +-- zuul/scheduler.py | 13 +++---- zuul/source/__init__.py | 5 +-- zuul/trigger/__init__.py | 5 +-- 22 files changed, 68 insertions(+), 100 deletions(-) diff --git a/requirements.txt b/requirements.txt index eda36c62d5..81f930ef47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,6 @@ gear>=0.9.0,<1.0.0 apscheduler>=3.0 PrettyTable>=0.6,<0.8 babel>=1.0 -six>=1.6.0 ansible>=2.0.0.1 kazoo sqlalchemy diff --git a/tests/base.py b/tests/base.py index 93b5785de9..d89469f874 100755 --- a/tests/base.py +++ b/tests/base.py @@ -15,24 +15,20 @@ # License for the specific language governing permissions and limitations # under the License. -from six.moves import configparser as ConfigParser +import configparser import datetime import gc import hashlib +import importlib +from io import StringIO import json import logging import os -from six.moves import queue as Queue -from six.moves import urllib +import queue import random import re import select import shutil -from six.moves import reload_module -try: - from cStringIO import StringIO -except Exception: - from six import StringIO import socket import string import subprocess @@ -42,6 +38,7 @@ import threading import traceback import time import uuid +import urllib import git @@ -463,7 +460,7 @@ class FakeGerritConnection(gerritconnection.GerritConnection): super(FakeGerritConnection, self).__init__(driver, connection_name, connection_config) - self.event_queue = Queue.Queue() + self.event_queue = queue.Queue() self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit') self.change_number = 0 self.changes = changes_db @@ -1373,8 +1370,8 @@ class FakeGearmanServer(gear.Server): ssl_ca=ssl_ca) def getJobForConnection(self, connection, peek=False): - for queue in [self.high_queue, self.normal_queue, self.low_queue]: - for job in queue: + for job_queue in [self.high_queue, self.normal_queue, self.low_queue]: + for job in job_queue: if not hasattr(job, 'waiting'): if job.name.startswith(b'executor:execute'): job.waiting = self.hold_jobs_in_queue @@ -1384,7 +1381,7 @@ class FakeGearmanServer(gear.Server): continue if job.name in connection.functions: if not peek: - queue.remove(job) + job_queue.remove(job) connection.related_jobs[job.handle] = job job.worker_connection = connection job.running = True @@ -1879,8 +1876,8 @@ class ZuulTestCase(BaseTestCase): os.environ['STATSD_PORT'] = str(self.statsd.port) self.statsd.start() # the statsd client object is configured in the statsd module import - reload_module(statsd) - reload_module(zuul.scheduler) + importlib.reload(statsd) + importlib.reload(zuul.scheduler) self.gearman_server = FakeGearmanServer(self.use_ssl) @@ -2008,7 +2005,7 @@ class ZuulTestCase(BaseTestCase): # This creates the per-test configuration object. It can be # overriden by subclasses, but should not need to be since it # obeys the config_file and tenant_config_file attributes. - self.config = ConfigParser.ConfigParser() + self.config = configparser.ConfigParser() self.config.read(os.path.join(FIXTURE_DIR, self.config_file)) if not self.setupSimpleLayout(): @@ -2383,12 +2380,12 @@ class ZuulTestCase(BaseTestCase): return True def eventQueuesEmpty(self): - for queue in self.event_queues: - yield queue.empty() + for event_queue in self.event_queues: + yield event_queue.empty() def eventQueuesJoin(self): - for queue in self.event_queues: - queue.join() + for event_queue in self.event_queues: + event_queue.join() def waitUntilSettled(self): self.log.debug("Waiting until settled...") @@ -2397,8 +2394,9 @@ class ZuulTestCase(BaseTestCase): if time.time() - start > self.wait_timeout: self.log.error("Timeout waiting for Zuul to settle") self.log.error("Queue status:") - for queue in self.event_queues: - self.log.error(" %s: %s" % (queue, queue.empty())) + for event_queue in self.event_queues: + self.log.error(" %s: %s" % + (event_queue, event_queue.empty())) self.log.error("All builds waiting: %s" % (self.areAllBuildsWaiting(),)) self.log.error("All builds reported: %s" % @@ -2457,11 +2455,12 @@ class ZuulTestCase(BaseTestCase): # Make sure there are no orphaned jobs for tenant in self.sched.abide.tenants.values(): for pipeline in tenant.layout.pipelines.values(): - for queue in pipeline.queues: - if len(queue.queue) != 0: + for pipeline_queue in pipeline.queues: + if len(pipeline_queue.queue) != 0: print('pipeline %s queue %s contents %s' % ( - pipeline.name, queue.name, queue.queue)) - self.assertEqual(len(queue.queue), 0, + pipeline.name, pipeline_queue.name, + pipeline_queue.queue)) + self.assertEqual(len(pipeline_queue.queue), 0, "Pipelines queues should be empty") def assertReportedStat(self, key, value=None, kind=None): diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index 2f9d272901..eb1796663d 100755 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -25,8 +25,8 @@ import time from unittest import skip import git -from six.moves import urllib import testtools +import urllib import zuul.change_matcher from zuul.driver.gerrit import gerritreporter diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index b2836ae586..da027c1df4 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -17,8 +17,8 @@ import os import json +import urllib -from six.moves import urllib import webob from tests.base import ZuulTestCase, FIXTURE_DIR diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py index 4865edd97b..e36b24e086 100644 --- a/tools/encrypt_secret.py +++ b/tools/encrypt_secret.py @@ -17,7 +17,7 @@ import os import subprocess import sys import tempfile -from six.moves import urllib +import urllib DESCRIPTION = """Encrypt a secret for Zuul. diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py index d31c5b8700..86101148b8 100755 --- a/zuul/cmd/__init__.py +++ b/zuul/cmd/__init__.py @@ -14,9 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -import six -from six.moves import configparser as ConfigParser +import configparser import extras +import io import logging import logging.config import os @@ -48,7 +48,7 @@ def stack_dump_handler(signum, frame): yappi.start() else: yappi.stop() - yappi_out = six.BytesIO() + yappi_out = io.BytesIO() yappi.get_func_stats().print_all(out=yappi_out) yappi.get_thread_stats().print_all(out=yappi_out) log.debug(yappi_out.getvalue()) @@ -69,7 +69,7 @@ class ZuulApp(object): return "Zuul version: %s" % zuul_version_info.release_string() def read_config(self): - self.config = ConfigParser.ConfigParser() + self.config = configparser.ConfigParser() if self.args.config: locations = [self.args.config] else: diff --git a/zuul/configloader.py b/zuul/configloader.py index 5e0fe65dfa..d4f7c43642 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -15,7 +15,6 @@ from contextlib import contextmanager import copy import os import logging -import six import pprint import textwrap @@ -427,7 +426,7 @@ class JobParser(object): setattr(job, a, conf[k]) if 'nodes' in conf: conf_nodes = conf['nodes'] - if isinstance(conf_nodes, six.string_types): + if isinstance(conf_nodes, str): # This references an existing named nodeset in the layout. ns = layout.nodesets[conf_nodes] else: @@ -576,7 +575,7 @@ class ProjectTemplateParser(object): def _parseJobList(tenant, layout, conf, source_context, start_mark, job_list): for conf_job in conf: - if isinstance(conf_job, six.string_types): + if isinstance(conf_job, str): attrs = dict(name=conf_job) elif isinstance(conf_job, dict): # A dictionary in a job tree may override params @@ -1007,7 +1006,7 @@ class TenantParser(object): @staticmethod def _getProject(source, conf, current_include): - if isinstance(conf, six.string_types): + if isinstance(conf, str): # Return a project object whether conf is a dict or a str project = source.getProject(conf) project_include = current_include @@ -1031,7 +1030,7 @@ class TenantParser(object): def _getProjects(source, conf, current_include): # Return a project object whether conf is a dict or a str projects = [] - if isinstance(conf, six.string_types): + if isinstance(conf, str): # A simple project name string projects.append(TenantParser._getProject( source, conf, current_include)) diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py index 90ab39c12f..3655115c3a 100644 --- a/zuul/connection/__init__.py +++ b/zuul/connection/__init__.py @@ -15,11 +15,9 @@ import abc import extras -import six -@six.add_metaclass(abc.ABCMeta) -class BaseConnection(object): +class BaseConnection(object, metaclass=abc.ABCMeta): """Base class for connections. A connection is a shared object that sources, triggers and reporters can diff --git a/zuul/driver/__init__.py b/zuul/driver/__init__.py index 0c3105d1be..b6c34e720f 100644 --- a/zuul/driver/__init__.py +++ b/zuul/driver/__init__.py @@ -14,11 +14,8 @@ import abc -import six - -@six.add_metaclass(abc.ABCMeta) -class Driver(object): +class Driver(object, metaclass=abc.ABCMeta): """A Driver is an extension component of Zuul that supports interfacing with a remote system. It can support any of the following interfaces (but must support at least one to be useful): @@ -80,8 +77,7 @@ class Driver(object): pass -@six.add_metaclass(abc.ABCMeta) -class ConnectionInterface(object): +class ConnectionInterface(object, metaclass=abc.ABCMeta): """The Connection interface. A driver which is able to supply a Connection should implement @@ -124,8 +120,7 @@ class ConnectionInterface(object): pass -@six.add_metaclass(abc.ABCMeta) -class TriggerInterface(object): +class TriggerInterface(object, metaclass=abc.ABCMeta): """The trigger interface. A driver which is able to supply a trigger should implement this @@ -167,8 +162,7 @@ class TriggerInterface(object): pass -@six.add_metaclass(abc.ABCMeta) -class SourceInterface(object): +class SourceInterface(object, metaclass=abc.ABCMeta): """The source interface to be implemented by a driver. A driver which is able to supply a Source should implement this @@ -216,8 +210,7 @@ class SourceInterface(object): pass -@six.add_metaclass(abc.ABCMeta) -class ReporterInterface(object): +class ReporterInterface(object, metaclass=abc.ABCMeta): """The reporter interface to be implemented by a driver. A driver which is able to supply a Reporter should implement this @@ -256,8 +249,7 @@ class ReporterInterface(object): pass -@six.add_metaclass(abc.ABCMeta) -class WrapperInterface(object): +class WrapperInterface(object, metaclass=abc.ABCMeta): """The wrapper interface to be implmeneted by a driver. A driver which wraps execution of commands executed by Zuul should diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py index e3a9866113..100263503f 100644 --- a/zuul/driver/bubblewrap/__init__.py +++ b/zuul/driver/bubblewrap/__init__.py @@ -19,11 +19,10 @@ import grp import logging import os import pwd +import shlex import subprocess import sys -from six.moves import shlex_quote - from zuul.driver import (Driver, WrapperInterface) @@ -144,7 +143,7 @@ class BubblewrapDriver(Driver, WrapperInterface): command = [x.format(**kwargs) for x in bwrap_command] self.log.debug("Bubblewrap command: %s", - " ".join(shlex_quote(c) for c in command)) + " ".join(shlex.quote(c) for c in command)) wrapped_popen = WrappedPopen(command, passwd_r, group_r) diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py index 9033ee632a..39a81bca43 100644 --- a/zuul/driver/gerrit/gerritconnection.py +++ b/zuul/driver/gerrit/gerritconnection.py @@ -18,11 +18,11 @@ import re import select import threading import time -from six.moves import queue as Queue -from six.moves import shlex_quote import paramiko import logging import pprint +import shlex +import queue import voluptuous as v from zuul.connection import BaseConnection @@ -260,7 +260,7 @@ class GerritConnection(BaseConnection): self.keyfile = self.connection_config.get('sshkey', None) self.keepalive = int(self.connection_config.get('keepalive', 60)) self.watcher_thread = None - self.event_queue = Queue.Queue() + self.event_queue = queue.Queue() self.client = None self.baseurl = self.connection_config.get('baseurl', @@ -606,7 +606,7 @@ class GerritConnection(BaseConnection): def review(self, project, change, message, action={}): cmd = 'gerrit review --project %s' % project if message: - cmd += ' --message %s' % shlex_quote(message) + cmd += ' --message %s' % shlex.quote(message) for key, val in action.items(): if val is True: cmd += ' --%s' % key diff --git a/zuul/driver/git/gitconnection.py b/zuul/driver/git/gitconnection.py index ca88d3fe21..f4fe7e5efb 100644 --- a/zuul/driver/git/gitconnection.py +++ b/zuul/driver/git/gitconnection.py @@ -14,7 +14,7 @@ # under the License. import logging -from six.moves import urllib +import urllib import voluptuous as v diff --git a/zuul/executor/server.py b/zuul/executor/server.py index f2aedbabb1..94678d768a 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -18,6 +18,7 @@ import logging import os import shutil import signal +import shlex import socket import subprocess import tempfile @@ -27,7 +28,6 @@ import traceback from zuul.lib.yamlutil import yaml import gear -from six.moves import shlex_quote import zuul.merger.merger import zuul.ansible @@ -1288,7 +1288,7 @@ class AnsibleJob(object): if self.aborted: return (self.RESULT_ABORTED, None) self.log.debug("Ansible command: ANSIBLE_CONFIG=%s %s", - config_file, " ".join(shlex_quote(c) for c in cmd)) + config_file, " ".join(shlex.quote(c) for c in cmd)) self.proc = popen( cmd, cwd=self.jobdir.work_root, diff --git a/zuul/lib/clonemapper.py b/zuul/lib/clonemapper.py index 308260e117..7423308026 100644 --- a/zuul/lib/clonemapper.py +++ b/zuul/lib/clonemapper.py @@ -19,8 +19,6 @@ import logging import os import re -import six - class CloneMapper(object): log = logging.getLogger("zuul.CloneMapper") @@ -58,17 +56,17 @@ class CloneMapper(object): raise Exception("Expansion error. Check error messages above") self.log.info("Mapping projects to workspace...") - for project, dest in six.iteritems(ret): + for project, dest in ret.items(): dest = os.path.normpath(os.path.join(workspace, dest[0])) ret[project] = dest self.log.info(" %s -> %s", project, dest) self.log.debug("Checking overlap in destination directories...") check = defaultdict(list) - for project, dest in six.iteritems(ret): + for project, dest in ret.items(): check[dest].append(project) - dupes = dict((d, p) for (d, p) in six.iteritems(check) if len(p) > 1) + dupes = dict((d, p) for (d, p) in check.items() if len(p) > 1) if dupes: raise Exception("Some projects share the same destination: %s", dupes) diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py index 3070be6161..3fcffbebf0 100644 --- a/zuul/lib/cloner.py +++ b/zuul/lib/cloner.py @@ -18,8 +18,6 @@ import logging import os import re -import six - from git import GitCommandError from zuul import exceptions from zuul.lib.clonemapper import CloneMapper @@ -72,7 +70,7 @@ class Cloner(object): dests = mapper.expand(workspace=self.workspace) self.log.info("Preparing %s repositories", len(dests)) - for project, dest in six.iteritems(dests): + for project, dest in dests.items(): self.prepareRepo(project, dest) self.log.info("Prepared all repositories") diff --git a/zuul/lib/commandsocket.py b/zuul/lib/commandsocket.py index ae62204a87..901291a6f9 100644 --- a/zuul/lib/commandsocket.py +++ b/zuul/lib/commandsocket.py @@ -18,7 +18,7 @@ import logging import os import socket import threading -from six.moves import queue +import queue class CommandSocket(object): diff --git a/zuul/model.py b/zuul/model.py index 62ed92e4d2..a89c6d13eb 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -21,9 +21,6 @@ import struct import time from uuid import uuid4 -import six - - MERGER_MERGE = 1 # "git merge" MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve" MERGER_CHERRY_PICK = 3 # "git cherry-pick" @@ -666,8 +663,7 @@ class PlaybookContext(object): path=self.path) -@six.add_metaclass(abc.ABCMeta) -class Role(object): +class Role(object, metaclass=abc.ABCMeta): """A reference to an ansible role.""" def __init__(self, target_name): diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py index 89830d5fee..0ac57669c0 100644 --- a/zuul/reporter/__init__.py +++ b/zuul/reporter/__init__.py @@ -15,11 +15,8 @@ import abc import logging -import six - -@six.add_metaclass(abc.ABCMeta) -class BaseReporter(object): +class BaseReporter(object, metaclass=abc.ABCMeta): """Base class for reporters. Defines the exact public methods that must be supplied. diff --git a/zuul/rpclistener.py b/zuul/rpclistener.py index 0079ab8c4a..c1ee50ef6a 100644 --- a/zuul/rpclistener.py +++ b/zuul/rpclistener.py @@ -19,7 +19,6 @@ import threading import traceback import gear -import six from zuul import model @@ -179,8 +178,7 @@ class RPCListener(object): # TODO: use args to filter by pipeline etc running_items = [] for tenant in self.sched.abide.tenants.values(): - for pipeline_name, pipeline in six.iteritems( - tenant.layout.pipelines): + for pipeline_name, pipeline in tenant.layout.pipelines.items(): for queue in pipeline.queues: for item in queue.queue: running_items.append(item.formatJSON()) diff --git a/zuul/scheduler.py b/zuul/scheduler.py index a63d270427..2076163135 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -20,8 +20,7 @@ import json import logging import os import pickle -import six -from six.moves import queue as Queue +import queue import socket import sys import threading @@ -49,7 +48,9 @@ class ManagementEvent(object): def wait(self, timeout=None): self._wait_event.wait(timeout) if self._exc_info: - six.reraise(*self._exc_info) + # http://python3porting.com/differences.html#raise + e, v, t = self._exc_info + raise e(v).with_traceback(t) return self._wait_event.is_set() @@ -217,9 +218,9 @@ class Scheduler(threading.Thread): self.triggers = dict() self.config = config - self.trigger_event_queue = Queue.Queue() - self.result_event_queue = Queue.Queue() - self.management_event_queue = Queue.Queue() + self.trigger_event_queue = queue.Queue() + self.result_event_queue = queue.Queue() + self.management_event_queue = queue.Queue() self.abide = model.Abide() if not testonly: diff --git a/zuul/source/__init__.py b/zuul/source/__init__.py index 68baf0e009..b37aeb4d1d 100644 --- a/zuul/source/__init__.py +++ b/zuul/source/__init__.py @@ -14,11 +14,8 @@ import abc -import six - -@six.add_metaclass(abc.ABCMeta) -class BaseSource(object): +class BaseSource(object, metaclass=abc.ABCMeta): """Base class for sources. A source class gives methods for fetching and updating changes. Each diff --git a/zuul/trigger/__init__.py b/zuul/trigger/__init__.py index a5406d6cbd..a67c99bf7c 100644 --- a/zuul/trigger/__init__.py +++ b/zuul/trigger/__init__.py @@ -14,11 +14,8 @@ import abc -import six - -@six.add_metaclass(abc.ABCMeta) -class BaseTrigger(object): +class BaseTrigger(object, metaclass=abc.ABCMeta): """Base class for triggers. Defines the exact public methods that must be supplied."""