Browse Source

Remove use of six library

It exists only for py2/py3 compat. We do not need it any more.

This will explicitly break Zuul v3 for python2, which is different than
simply ceasing to test it and no longer declaring we support it. Since
we're not testing it any longer, it's bound to degrade overtime without
us noticing, so hopefully a clean and explicit break will prevent people
from running under python2 and it working for a minute, then breaking
later.

Change-Id: Ia16bb399a2869ab37a183f3f2197275bb3acafee
changes/35/475135/4
Monty Taylor 4 years ago
parent
commit
b934c1a052
No known key found for this signature in database GPG Key ID: 7BAE94BC7141A594
  1. 1
      requirements.txt
  2. 49
      tests/base.py
  3. 2
      tests/unit/test_scheduler.py
  4. 2
      tests/unit/test_webapp.py
  5. 2
      tools/encrypt_secret.py
  6. 8
      zuul/cmd/__init__.py
  7. 9
      zuul/configloader.py
  8. 4
      zuul/connection/__init__.py
  9. 20
      zuul/driver/__init__.py
  10. 5
      zuul/driver/bubblewrap/__init__.py
  11. 8
      zuul/driver/gerrit/gerritconnection.py
  12. 2
      zuul/driver/git/gitconnection.py
  13. 4
      zuul/executor/server.py
  14. 8
      zuul/lib/clonemapper.py
  15. 4
      zuul/lib/cloner.py
  16. 2
      zuul/lib/commandsocket.py
  17. 6
      zuul/model.py
  18. 5
      zuul/reporter/__init__.py
  19. 4
      zuul/rpclistener.py
  20. 13
      zuul/scheduler.py
  21. 5
      zuul/source/__init__.py
  22. 5
      zuul/trigger/__init__.py

1
requirements.txt

@ -16,7 +16,6 @@ gear>=0.9.0,<1.0.0
apscheduler>=3.0
PrettyTable>=0.6,<0.8
babel>=1.0
six>=1.6.0
ansible>=2.0.0.1
kazoo
sqlalchemy

49
tests/base.py

@ -15,24 +15,20 @@
# License for the specific language governing permissions and limitations
# under the License.
from six.moves import configparser as ConfigParser
import configparser
import datetime
import gc
import hashlib
import importlib
from io import StringIO
import json
import logging
import os
from six.moves import queue as Queue
from six.moves import urllib
import queue
import random
import re
import select
import shutil
from six.moves import reload_module
try:
from cStringIO import StringIO
except Exception:
from six import StringIO
import socket
import string
import subprocess
@ -42,6 +38,7 @@ import threading
import traceback
import time
import uuid
import urllib
import git
@ -463,7 +460,7 @@ class FakeGerritConnection(gerritconnection.GerritConnection):
super(FakeGerritConnection, self).__init__(driver, connection_name,
connection_config)
self.event_queue = Queue.Queue()
self.event_queue = queue.Queue()
self.fixture_dir = os.path.join(FIXTURE_DIR, 'gerrit')
self.change_number = 0
self.changes = changes_db
@ -1373,8 +1370,8 @@ class FakeGearmanServer(gear.Server):
ssl_ca=ssl_ca)
def getJobForConnection(self, connection, peek=False):
for queue in [self.high_queue, self.normal_queue, self.low_queue]:
for job in queue:
for job_queue in [self.high_queue, self.normal_queue, self.low_queue]:
for job in job_queue:
if not hasattr(job, 'waiting'):
if job.name.startswith(b'executor:execute'):
job.waiting = self.hold_jobs_in_queue
@ -1384,7 +1381,7 @@ class FakeGearmanServer(gear.Server):
continue
if job.name in connection.functions:
if not peek:
queue.remove(job)
job_queue.remove(job)
connection.related_jobs[job.handle] = job
job.worker_connection = connection
job.running = True
@ -1879,8 +1876,8 @@ class ZuulTestCase(BaseTestCase):
os.environ['STATSD_PORT'] = str(self.statsd.port)
self.statsd.start()
# the statsd client object is configured in the statsd module import
reload_module(statsd)
reload_module(zuul.scheduler)
importlib.reload(statsd)
importlib.reload(zuul.scheduler)
self.gearman_server = FakeGearmanServer(self.use_ssl)
@ -2008,7 +2005,7 @@ class ZuulTestCase(BaseTestCase):
# This creates the per-test configuration object. It can be
# overriden by subclasses, but should not need to be since it
# obeys the config_file and tenant_config_file attributes.
self.config = ConfigParser.ConfigParser()
self.config = configparser.ConfigParser()
self.config.read(os.path.join(FIXTURE_DIR, self.config_file))
if not self.setupSimpleLayout():
@ -2383,12 +2380,12 @@ class ZuulTestCase(BaseTestCase):
return True
def eventQueuesEmpty(self):
for queue in self.event_queues:
yield queue.empty()
for event_queue in self.event_queues:
yield event_queue.empty()
def eventQueuesJoin(self):
for queue in self.event_queues:
queue.join()
for event_queue in self.event_queues:
event_queue.join()
def waitUntilSettled(self):
self.log.debug("Waiting until settled...")
@ -2397,8 +2394,9 @@ class ZuulTestCase(BaseTestCase):
if time.time() - start > self.wait_timeout:
self.log.error("Timeout waiting for Zuul to settle")
self.log.error("Queue status:")
for queue in self.event_queues:
self.log.error(" %s: %s" % (queue, queue.empty()))
for event_queue in self.event_queues:
self.log.error(" %s: %s" %
(event_queue, event_queue.empty()))
self.log.error("All builds waiting: %s" %
(self.areAllBuildsWaiting(),))
self.log.error("All builds reported: %s" %
@ -2457,11 +2455,12 @@ class ZuulTestCase(BaseTestCase):
# Make sure there are no orphaned jobs
for tenant in self.sched.abide.tenants.values():
for pipeline in tenant.layout.pipelines.values():
for queue in pipeline.queues:
if len(queue.queue) != 0:
for pipeline_queue in pipeline.queues:
if len(pipeline_queue.queue) != 0:
print('pipeline %s queue %s contents %s' % (
pipeline.name, queue.name, queue.queue))
self.assertEqual(len(queue.queue), 0,
pipeline.name, pipeline_queue.name,
pipeline_queue.queue))
self.assertEqual(len(pipeline_queue.queue), 0,
"Pipelines queues should be empty")
def assertReportedStat(self, key, value=None, kind=None):

2
tests/unit/test_scheduler.py

@ -25,8 +25,8 @@ import time
from unittest import skip
import git
from six.moves import urllib
import testtools
import urllib
import zuul.change_matcher
from zuul.driver.gerrit import gerritreporter

2
tests/unit/test_webapp.py

@ -17,8 +17,8 @@
import os
import json
import urllib
from six.moves import urllib
import webob
from tests.base import ZuulTestCase, FIXTURE_DIR

2
tools/encrypt_secret.py

@ -17,7 +17,7 @@ import os
import subprocess
import sys
import tempfile
from six.moves import urllib
import urllib
DESCRIPTION = """Encrypt a secret for Zuul.

8
zuul/cmd/__init__.py

@ -14,9 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
from six.moves import configparser as ConfigParser
import configparser
import extras
import io
import logging
import logging.config
import os
@ -48,7 +48,7 @@ def stack_dump_handler(signum, frame):
yappi.start()
else:
yappi.stop()
yappi_out = six.BytesIO()
yappi_out = io.BytesIO()
yappi.get_func_stats().print_all(out=yappi_out)
yappi.get_thread_stats().print_all(out=yappi_out)
log.debug(yappi_out.getvalue())
@ -69,7 +69,7 @@ class ZuulApp(object):
return "Zuul version: %s" % zuul_version_info.release_string()
def read_config(self):
self.config = ConfigParser.ConfigParser()
self.config = configparser.ConfigParser()
if self.args.config:
locations = [self.args.config]
else:

9
zuul/configloader.py

@ -15,7 +15,6 @@ from contextlib import contextmanager
import copy
import os
import logging
import six
import pprint
import textwrap
@ -427,7 +426,7 @@ class JobParser(object):
setattr(job, a, conf[k])
if 'nodes' in conf:
conf_nodes = conf['nodes']
if isinstance(conf_nodes, six.string_types):
if isinstance(conf_nodes, str):
# This references an existing named nodeset in the layout.
ns = layout.nodesets[conf_nodes]
else:
@ -576,7 +575,7 @@ class ProjectTemplateParser(object):
def _parseJobList(tenant, layout, conf, source_context,
start_mark, job_list):
for conf_job in conf:
if isinstance(conf_job, six.string_types):
if isinstance(conf_job, str):
attrs = dict(name=conf_job)
elif isinstance(conf_job, dict):
# A dictionary in a job tree may override params
@ -1007,7 +1006,7 @@ class TenantParser(object):
@staticmethod
def _getProject(source, conf, current_include):
if isinstance(conf, six.string_types):
if isinstance(conf, str):
# Return a project object whether conf is a dict or a str
project = source.getProject(conf)
project_include = current_include
@ -1031,7 +1030,7 @@ class TenantParser(object):
def _getProjects(source, conf, current_include):
# Return a project object whether conf is a dict or a str
projects = []
if isinstance(conf, six.string_types):
if isinstance(conf, str):
# A simple project name string
projects.append(TenantParser._getProject(
source, conf, current_include))

4
zuul/connection/__init__.py

@ -15,11 +15,9 @@
import abc
import extras
import six
@six.add_metaclass(abc.ABCMeta)
class BaseConnection(object):
class BaseConnection(object, metaclass=abc.ABCMeta):
"""Base class for connections.
A connection is a shared object that sources, triggers and reporters can

20
zuul/driver/__init__.py

@ -14,11 +14,8 @@
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class Driver(object):
class Driver(object, metaclass=abc.ABCMeta):
"""A Driver is an extension component of Zuul that supports
interfacing with a remote system. It can support any of the following
interfaces (but must support at least one to be useful):
@ -80,8 +77,7 @@ class Driver(object):
pass
@six.add_metaclass(abc.ABCMeta)
class ConnectionInterface(object):
class ConnectionInterface(object, metaclass=abc.ABCMeta):
"""The Connection interface.
A driver which is able to supply a Connection should implement
@ -124,8 +120,7 @@ class ConnectionInterface(object):
pass
@six.add_metaclass(abc.ABCMeta)
class TriggerInterface(object):
class TriggerInterface(object, metaclass=abc.ABCMeta):
"""The trigger interface.
A driver which is able to supply a trigger should implement this
@ -167,8 +162,7 @@ class TriggerInterface(object):
pass
@six.add_metaclass(abc.ABCMeta)
class SourceInterface(object):
class SourceInterface(object, metaclass=abc.ABCMeta):
"""The source interface to be implemented by a driver.
A driver which is able to supply a Source should implement this
@ -216,8 +210,7 @@ class SourceInterface(object):
pass
@six.add_metaclass(abc.ABCMeta)
class ReporterInterface(object):
class ReporterInterface(object, metaclass=abc.ABCMeta):
"""The reporter interface to be implemented by a driver.
A driver which is able to supply a Reporter should implement this
@ -256,8 +249,7 @@ class ReporterInterface(object):
pass
@six.add_metaclass(abc.ABCMeta)
class WrapperInterface(object):
class WrapperInterface(object, metaclass=abc.ABCMeta):
"""The wrapper interface to be implmeneted by a driver.
A driver which wraps execution of commands executed by Zuul should

5
zuul/driver/bubblewrap/__init__.py

@ -19,11 +19,10 @@ import grp
import logging
import os
import pwd
import shlex
import subprocess
import sys
from six.moves import shlex_quote
from zuul.driver import (Driver, WrapperInterface)
@ -144,7 +143,7 @@ class BubblewrapDriver(Driver, WrapperInterface):
command = [x.format(**kwargs) for x in bwrap_command]
self.log.debug("Bubblewrap command: %s",
" ".join(shlex_quote(c) for c in command))
" ".join(shlex.quote(c) for c in command))
wrapped_popen = WrappedPopen(command, passwd_r, group_r)

8
zuul/driver/gerrit/gerritconnection.py

@ -18,11 +18,11 @@ import re
import select
import threading
import time
from six.moves import queue as Queue
from six.moves import shlex_quote
import paramiko
import logging
import pprint
import shlex
import queue
import voluptuous as v
from zuul.connection import BaseConnection
@ -260,7 +260,7 @@ class GerritConnection(BaseConnection):
self.keyfile = self.connection_config.get('sshkey', None)
self.keepalive = int(self.connection_config.get('keepalive', 60))
self.watcher_thread = None
self.event_queue = Queue.Queue()
self.event_queue = queue.Queue()
self.client = None
self.baseurl = self.connection_config.get('baseurl',
@ -606,7 +606,7 @@ class GerritConnection(BaseConnection):
def review(self, project, change, message, action={}):
cmd = 'gerrit review --project %s' % project
if message:
cmd += ' --message %s' % shlex_quote(message)
cmd += ' --message %s' % shlex.quote(message)
for key, val in action.items():
if val is True:
cmd += ' --%s' % key

2
zuul/driver/git/gitconnection.py

@ -14,7 +14,7 @@
# under the License.
import logging
from six.moves import urllib
import urllib
import voluptuous as v

4
zuul/executor/server.py

@ -18,6 +18,7 @@ import logging
import os
import shutil
import signal
import shlex
import socket
import subprocess
import tempfile
@ -27,7 +28,6 @@ import traceback
from zuul.lib.yamlutil import yaml
import gear
from six.moves import shlex_quote
import zuul.merger.merger
import zuul.ansible
@ -1288,7 +1288,7 @@ class AnsibleJob(object):
if self.aborted:
return (self.RESULT_ABORTED, None)
self.log.debug("Ansible command: ANSIBLE_CONFIG=%s %s",
config_file, " ".join(shlex_quote(c) for c in cmd))
config_file, " ".join(shlex.quote(c) for c in cmd))
self.proc = popen(
cmd,
cwd=self.jobdir.work_root,

8
zuul/lib/clonemapper.py

@ -19,8 +19,6 @@ import logging
import os
import re
import six
class CloneMapper(object):
log = logging.getLogger("zuul.CloneMapper")
@ -58,17 +56,17 @@ class CloneMapper(object):
raise Exception("Expansion error. Check error messages above")
self.log.info("Mapping projects to workspace...")
for project, dest in six.iteritems(ret):
for project, dest in ret.items():
dest = os.path.normpath(os.path.join(workspace, dest[0]))
ret[project] = dest
self.log.info(" %s -> %s", project, dest)
self.log.debug("Checking overlap in destination directories...")
check = defaultdict(list)
for project, dest in six.iteritems(ret):
for project, dest in ret.items():
check[dest].append(project)
dupes = dict((d, p) for (d, p) in six.iteritems(check) if len(p) > 1)
dupes = dict((d, p) for (d, p) in check.items() if len(p) > 1)
if dupes:
raise Exception("Some projects share the same destination: %s",
dupes)

4
zuul/lib/cloner.py

@ -18,8 +18,6 @@ import logging
import os
import re
import six
from git import GitCommandError
from zuul import exceptions
from zuul.lib.clonemapper import CloneMapper
@ -72,7 +70,7 @@ class Cloner(object):
dests = mapper.expand(workspace=self.workspace)
self.log.info("Preparing %s repositories", len(dests))
for project, dest in six.iteritems(dests):
for project, dest in dests.items():
self.prepareRepo(project, dest)
self.log.info("Prepared all repositories")

2
zuul/lib/commandsocket.py

@ -18,7 +18,7 @@ import logging
import os
import socket
import threading
from six.moves import queue
import queue
class CommandSocket(object):

6
zuul/model.py

@ -21,9 +21,6 @@ import struct
import time
from uuid import uuid4
import six
MERGER_MERGE = 1 # "git merge"
MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
MERGER_CHERRY_PICK = 3 # "git cherry-pick"
@ -666,8 +663,7 @@ class PlaybookContext(object):
path=self.path)
@six.add_metaclass(abc.ABCMeta)
class Role(object):
class Role(object, metaclass=abc.ABCMeta):
"""A reference to an ansible role."""
def __init__(self, target_name):

5
zuul/reporter/__init__.py

@ -15,11 +15,8 @@
import abc
import logging
import six
@six.add_metaclass(abc.ABCMeta)
class BaseReporter(object):
class BaseReporter(object, metaclass=abc.ABCMeta):
"""Base class for reporters.
Defines the exact public methods that must be supplied.

4
zuul/rpclistener.py

@ -19,7 +19,6 @@ import threading
import traceback
import gear
import six
from zuul import model
@ -179,8 +178,7 @@ class RPCListener(object):
# TODO: use args to filter by pipeline etc
running_items = []
for tenant in self.sched.abide.tenants.values():
for pipeline_name, pipeline in six.iteritems(
tenant.layout.pipelines):
for pipeline_name, pipeline in tenant.layout.pipelines.items():
for queue in pipeline.queues:
for item in queue.queue:
running_items.append(item.formatJSON())

13
zuul/scheduler.py

@ -20,8 +20,7 @@ import json
import logging
import os
import pickle
import six
from six.moves import queue as Queue
import queue
import socket
import sys
import threading
@ -49,7 +48,9 @@ class ManagementEvent(object):
def wait(self, timeout=None):
self._wait_event.wait(timeout)
if self._exc_info:
six.reraise(*self._exc_info)
# http://python3porting.com/differences.html#raise
e, v, t = self._exc_info
raise e(v).with_traceback(t)
return self._wait_event.is_set()
@ -217,9 +218,9 @@ class Scheduler(threading.Thread):
self.triggers = dict()
self.config = config
self.trigger_event_queue = Queue.Queue()
self.result_event_queue = Queue.Queue()
self.management_event_queue = Queue.Queue()
self.trigger_event_queue = queue.Queue()
self.result_event_queue = queue.Queue()
self.management_event_queue = queue.Queue()
self.abide = model.Abide()
if not testonly:

5
zuul/source/__init__.py

@ -14,11 +14,8 @@
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class BaseSource(object):
class BaseSource(object, metaclass=abc.ABCMeta):
"""Base class for sources.
A source class gives methods for fetching and updating changes. Each

5
zuul/trigger/__init__.py

@ -14,11 +14,8 @@
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class BaseTrigger(object):
class BaseTrigger(object, metaclass=abc.ABCMeta):
"""Base class for triggers.
Defines the exact public methods that must be supplied."""

Loading…
Cancel
Save