From 405ff049ff349b94830e625366617ab4a2d4ad51 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 2 Sep 2014 15:37:59 -0700 Subject: [PATCH] Revert "Some tweaks to get closer to Python 3 compat" This reverts commit 1d7647fa857fa718af814f3038d538d758c35201. This change altered the xml output (by forcing it to fail) which a backward compatible change should not do. Revert it in order to get this compat change in without breaking that output and test. Change-Id: I20f66fb1bd9c70a0debbdd5eebacf6ec5d0f5df9 --- jenkins_jobs/builder.py | 22 ++++++++---------- jenkins_jobs/cmd.py | 13 +++++------ jenkins_jobs/local_yaml.py | 2 +- jenkins_jobs/modules/builders.py | 2 +- jenkins_jobs/modules/hipchat_notif.py | 6 ++--- jenkins_jobs/modules/publishers.py | 14 +++++------- jenkins_jobs/modules/scm.py | 10 ++++---- jenkins_jobs/modules/triggers.py | 4 ++-- jenkins_jobs/modules/zuul.py | 6 ++--- requirements.txt | 3 +-- tests/base.py | 13 ++++++----- tests/cmd/test_cmd.py | 33 ++++++++++++++------------- 12 files changed, 60 insertions(+), 68 deletions(-) diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index cbf95f2fa..463c03951 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -17,7 +17,6 @@ import errno import os -import operator import sys import hashlib import yaml @@ -32,9 +31,8 @@ import logging import copy import itertools import fnmatch -import six from jenkins_jobs.errors import JenkinsJobsException -import jenkins_jobs.local_yaml as local_yaml +import local_yaml logger = logging.getLogger(__name__) MAGIC_MANAGE_STRING = "" @@ -84,7 +82,7 @@ def deep_format(obj, paramdict): # limitations on the values in paramdict - the post-format result must # still be valid YAML (so substituting-in a string containing quotes, for # example, is problematic). - if hasattr(obj, 'format'): + if isinstance(obj, basestring): try: result = re.match('^{obj:(?P\w+)}$', obj) if result is not None: @@ -144,7 +142,7 @@ class YamlParser(object): " not a {cls}".format(fname=getattr(fp, 'name', fp), cls=type(data))) for item in data: - cls, dfn = next(iter(item.items())) + cls, dfn = item.items()[0] group = self.data.get(cls, {}) if len(item.items()) > 1: n = None @@ -211,7 +209,7 @@ class YamlParser(object): for jobspec in project.get('jobs', []): if isinstance(jobspec, dict): # Singleton dict containing dict of job-specific params - jobname, jobparams = jobspec.popitem() + jobname, jobparams = jobspec.items()[0] if not isinstance(jobparams, dict): jobparams = {} else: @@ -227,7 +225,7 @@ class YamlParser(object): for group_jobspec in group['jobs']: if isinstance(group_jobspec, dict): group_jobname, group_jobparams = \ - group_jobspec.popitem() + group_jobspec.items()[0] if not isinstance(group_jobparams, dict): group_jobparams = {} else: @@ -277,7 +275,7 @@ class YamlParser(object): expanded_values = {} for (k, v) in values: if isinstance(v, dict): - inner_key = next(iter(v)) + inner_key = v.iterkeys().next() expanded_values[k] = inner_key expanded_values.update(v[inner_key]) else: @@ -297,8 +295,6 @@ class YamlParser(object): # us guarantee a group of parameters will not be added a # second time. uniq = json.dumps(expanded, sort_keys=True) - if six.PY3: - uniq = uniq.encode('utf-8') checksum = hashlib.md5(uniq).hexdigest() # Lookup the checksum @@ -368,7 +364,7 @@ class ModuleRegistry(object): Mod = entrypoint.load() mod = Mod(self) self.modules.append(mod) - self.modules.sort(key=operator.attrgetter('sequence')) + self.modules.sort(lambda a, b: cmp(a.sequence, b.sequence)) if mod.component_type is not None: self.modules_by_component_type[mod.component_type] = mod @@ -412,7 +408,7 @@ class ModuleRegistry(object): if isinstance(component, dict): # The component is a singleton dictionary of name: dict(args) - name, component_data = next(iter(component.items())) + name, component_data = component.items()[0] if template_data: # Template data contains values that should be interpolated # into the component definition @@ -614,7 +610,7 @@ class Builder(object): self.load_files(input_fn) self.parser.generateXML(names) - self.parser.jobs.sort(key=operator.attrgetter('name')) + self.parser.jobs.sort(lambda a, b: cmp(a.name, b.name)) for job in self.parser.jobs: if names and not matches(job.name, names): diff --git a/jenkins_jobs/cmd.py b/jenkins_jobs/cmd.py index daa91363c..097c8484b 100755 --- a/jenkins_jobs/cmd.py +++ b/jenkins_jobs/cmd.py @@ -14,11 +14,12 @@ # under the License. import argparse -from six.moves import configparser, StringIO +import ConfigParser import logging import os import platform import sys +import cStringIO from jenkins_jobs.builder import Builder from jenkins_jobs.errors import JenkinsJobsException @@ -94,8 +95,6 @@ def main(argv=None): parser = create_parser() options = parser.parse_args(argv) - if not options.command: - parser.error("Must specify a 'command' to be performed") if (options.log_level is not None): options.log_level = getattr(logging, options.log_level.upper(), logger.getEffectiveLevel()) @@ -116,9 +115,9 @@ def setup_config_settings(options): 'jenkins_jobs.ini') if os.path.isfile(localconf): conf = localconf - config = configparser.ConfigParser() + config = ConfigParser.ConfigParser() ## Load default config always - config.readfp(StringIO(DEFAULT_CONF)) + config.readfp(cStringIO.StringIO(DEFAULT_CONF)) if os.path.isfile(conf): logger.debug("Reading config from {0}".format(conf)) conffp = open(conf, 'r') @@ -153,11 +152,11 @@ def execute(options, config): # https://bugs.launchpad.net/openstack-ci/+bug/1259631 try: user = config.get('jenkins', 'user') - except (TypeError, configparser.NoOptionError): + except (TypeError, ConfigParser.NoOptionError): user = None try: password = config.get('jenkins', 'password') - except (TypeError, configparser.NoOptionError): + except (TypeError, ConfigParser.NoOptionError): password = None builder = Builder(config.get('jenkins', 'url'), diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index c7f57ec2b..f66ad4374 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -163,7 +163,7 @@ class LocalLoader(OrderedConstructor, yaml.Loader): self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, type(self).construct_yaml_map) - if hasattr(self.stream, 'name'): + if isinstance(self.stream, file): self.search_path.add(os.path.normpath( os.path.dirname(self.stream.name))) self.search_path.add(os.path.normpath(os.path.curdir)) diff --git a/jenkins_jobs/modules/builders.py b/jenkins_jobs/modules/builders.py index 4ace5d463..72c19b8c4 100644 --- a/jenkins_jobs/modules/builders.py +++ b/jenkins_jobs/modules/builders.py @@ -227,7 +227,7 @@ def ant(parser, xml_parent, data): if type(data) is str: # Support for short form: -ant: "target" data = {'targets': data} - for setting, value in sorted(data.items()): + for setting, value in sorted(data.iteritems()): if setting == 'targets': targets = XML.SubElement(ant, 'targets') targets.text = value diff --git a/jenkins_jobs/modules/hipchat_notif.py b/jenkins_jobs/modules/hipchat_notif.py index 5ae825060..988ca4c87 100644 --- a/jenkins_jobs/modules/hipchat_notif.py +++ b/jenkins_jobs/modules/hipchat_notif.py @@ -46,7 +46,7 @@ import xml.etree.ElementTree as XML import jenkins_jobs.modules.base import jenkins_jobs.errors import logging -from six.moves import configparser +import ConfigParser import sys logger = logging.getLogger(__name__) @@ -73,8 +73,8 @@ class HipChat(jenkins_jobs.modules.base.Base): if self.authToken == '': raise jenkins_jobs.errors.JenkinsJobsException( "Hipchat authtoken must not be a blank string") - except (configparser.NoSectionError, - jenkins_jobs.errors.JenkinsJobsException) as e: + except (ConfigParser.NoSectionError, + jenkins_jobs.errors.JenkinsJobsException), e: logger.fatal("The configuration file needs a hipchat section" + " containing authtoken:\n{0}".format(e)) sys.exit(1) diff --git a/jenkins_jobs/modules/publishers.py b/jenkins_jobs/modules/publishers.py index 41eaf7312..d3f683f54 100644 --- a/jenkins_jobs/modules/publishers.py +++ b/jenkins_jobs/modules/publishers.py @@ -439,7 +439,7 @@ def cloverphp(parser, xml_parent, data): metrics = data.get('metric-targets', []) # list of dicts to dict - metrics = dict(kv for m in metrics for kv in m.items()) + metrics = dict(kv for m in metrics for kv in m.iteritems()) # Populate defaults whenever nothing has been filled by user. for default in default_metrics.keys(): @@ -889,7 +889,7 @@ def xunit(parser, xml_parent, data): supported_types = [] for configured_type in data['types']: - type_name = next(iter(configured_type.keys())) + type_name = configured_type.keys()[0] if type_name not in implemented_types: logger.warn("Requested xUnit type '%s' is not yet supported", type_name) @@ -900,7 +900,7 @@ def xunit(parser, xml_parent, data): # Generate XML for each of the supported framework types xmltypes = XML.SubElement(xunit, 'types') for supported_type in supported_types: - framework_name = next(iter(supported_type.keys())) + framework_name = supported_type.keys()[0] xmlframework = XML.SubElement(xmltypes, types_to_plugin_types[framework_name]) @@ -924,10 +924,9 @@ def xunit(parser, xml_parent, data): "Unrecognized threshold, should be 'failed' or 'skipped'") continue elname = "org.jenkinsci.plugins.xunit.threshold.%sThreshold" \ - % next(iter(t.keys())).title() + % t.keys()[0].title() el = XML.SubElement(xmlthresholds, elname) - for threshold_name, threshold_value in \ - next(iter(t.values())).items(): + for threshold_name, threshold_value in t.values()[0].items(): # Normalize and craft the element name for this threshold elname = "%sThreshold" % threshold_name.lower().replace( 'new', 'New') @@ -3510,8 +3509,7 @@ def ruby_metrics(parser, xml_parent, data): XML.SubElement(el, 'metric').text = 'TOTAL_COVERAGE' else: XML.SubElement(el, 'metric').text = 'CODE_COVERAGE' - for threshold_name, threshold_value in \ - next(iter(t.values())).items(): + for threshold_name, threshold_value in t.values()[0].items(): elname = threshold_name.lower() XML.SubElement(el, elname).text = str(threshold_value) else: diff --git a/jenkins_jobs/modules/scm.py b/jenkins_jobs/modules/scm.py index 81a78ca9f..89e599b10 100644 --- a/jenkins_jobs/modules/scm.py +++ b/jenkins_jobs/modules/scm.py @@ -161,9 +161,9 @@ remoteName/\*') data['remotes'] = [{data.get('name', 'origin'): data.copy()}] for remoteData in data['remotes']: huser = XML.SubElement(user, 'hudson.plugins.git.UserRemoteConfig') - remoteName = next(iter(remoteData.keys())) + remoteName = remoteData.keys()[0] XML.SubElement(huser, 'name').text = remoteName - remoteParams = next(iter(remoteData.values())) + remoteParams = remoteData.values()[0] if 'refspec' in remoteParams: refspec = remoteParams['refspec'] else: @@ -368,7 +368,7 @@ def store(parser, xml_parent, data): pundles = XML.SubElement(scm, 'pundles') for pundle_spec in pundle_specs: pundle = XML.SubElement(pundles, '{0}.PundleSpec'.format(namespace)) - pundle_type = next(iter(pundle_spec)) + pundle_type = pundle_spec.keys()[0] pundle_name = pundle_spec[pundle_type] if pundle_type not in valid_pundle_types: raise JenkinsJobsException( @@ -507,9 +507,9 @@ def tfs(parser, xml_parent, data): server. :arg str login: The user name that is registered on the server. The user name must contain the name and the domain name. Entered as - domain\\\\user or user\@domain (optional). + domain\\\user or user\@domain (optional). **NOTE**: You must enter in at least two slashes for the - domain\\\\user format in JJB YAML. It will be rendered normally. + domain\\\user format in JJB YAML. It will be rendered normally. :arg str use-update: If true, Hudson will not delete the workspace at end of each build. This causes the artifacts from the previous build to remain when a new build starts. (default true) diff --git a/jenkins_jobs/modules/triggers.py b/jenkins_jobs/modules/triggers.py index 65f74607e..a7c846fba 100644 --- a/jenkins_jobs/modules/triggers.py +++ b/jenkins_jobs/modules/triggers.py @@ -91,7 +91,7 @@ def build_gerrit_triggers(xml_parent, data): 'hudsontrigger.events' trigger_on_events = XML.SubElement(xml_parent, 'triggerOnEvents') - for config_key, tag_name in available_simple_triggers.items(): + for config_key, tag_name in available_simple_triggers.iteritems(): if data.get(config_key, False): XML.SubElement(trigger_on_events, '%s.%s' % (tag_namespace, tag_name)) @@ -453,7 +453,7 @@ def pollurl(parser, xml_parent, data): str(bool(check_content)).lower() content_types = XML.SubElement(entry, 'contentTypes') for entry in check_content: - type_name = next(iter(entry.keys())) + type_name = entry.keys()[0] if type_name not in valid_content_types: raise JenkinsJobsException('check-content must be one of : %s' % ', '.join(valid_content_types. diff --git a/jenkins_jobs/modules/zuul.py b/jenkins_jobs/modules/zuul.py index 79b428f63..cd823039a 100644 --- a/jenkins_jobs/modules/zuul.py +++ b/jenkins_jobs/modules/zuul.py @@ -35,8 +35,6 @@ The above URL is the default. http://ci.openstack.org/zuul/launchers.html#zuul-parameters """ -import itertools - def zuul(): """yaml: zuul @@ -154,8 +152,8 @@ class Zuul(jenkins_jobs.modules.base.Base): def handle_data(self, parser): changed = False - jobs = itertools.chain(parser.data.get('job', {}).values(), - parser.data.get('job-template', {}).values()) + jobs = (parser.data.get('job', {}).values() + + parser.data.get('job-template', {}).values()) for job in jobs: triggers = job.get('triggers') if not triggers: diff --git a/requirements.txt b/requirements.txt index 30110f646..cc7e45d8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ argparse -six>=1.5.2 PyYAML -python-jenkins>=0.3.3 +python-jenkins pbr>=0.8.2,<1.0 diff --git a/tests/base.py b/tests/base.py index 13f2bf7b9..d7e71817e 100644 --- a/tests/base.py +++ b/tests/base.py @@ -26,7 +26,7 @@ import json import operator import testtools import xml.etree.ElementTree as XML -from six.moves import configparser +from ConfigParser import ConfigParser import jenkins_jobs.local_yaml as yaml from jenkins_jobs.builder import XmlJob, YamlParser, ModuleRegistry from jenkins_jobs.modules import (project_flow, @@ -86,7 +86,7 @@ class BaseTestCase(object): def _read_yaml_content(self): yaml_filepath = os.path.join(self.fixtures_path, self.in_filename) - with open(yaml_filepath, 'r') as yaml_file: + with file(yaml_filepath, 'r') as yaml_file: yaml_content = yaml.load(yaml_file) return yaml_content @@ -118,7 +118,8 @@ class BaseTestCase(object): pub.gen_xml(parser, xml_project, yaml_content) # Prettify generated XML - pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8') + pretty_xml = unicode(XmlJob(xml_project, 'fixturejob').output(), + 'utf-8') self.assertThat( pretty_xml, @@ -136,7 +137,7 @@ class SingleJobTestCase(BaseTestCase): yaml_filepath = os.path.join(self.fixtures_path, self.in_filename) if self.conf_filename: - config = configparser.ConfigParser() + config = ConfigParser() conf_filepath = os.path.join(self.fixtures_path, self.conf_filename) config.readfp(open(conf_filepath)) @@ -151,8 +152,8 @@ class SingleJobTestCase(BaseTestCase): parser.jobs.sort(key=operator.attrgetter('name')) # Prettify generated XML - pretty_xml = u"\n".join(job.output().decode('utf-8') - for job in parser.jobs) + pretty_xml = unicode("\n".join(job.output() for job in parser.jobs), + 'utf-8') self.assertThat( pretty_xml, diff --git a/tests/cmd/test_cmd.py b/tests/cmd/test_cmd.py index 5e40a8fb9..e243796e9 100644 --- a/tests/cmd/test_cmd.py +++ b/tests/cmd/test_cmd.py @@ -1,6 +1,6 @@ import os -from six.moves import configparser, StringIO -import io +import ConfigParser +import cStringIO import codecs import mock import testtools @@ -22,15 +22,15 @@ class CmdTests(testtools.TestCase): User passes no args, should fail with SystemExit """ with mock.patch('sys.stderr'): - self.assertRaises(SystemExit, cmd.main, []) + self.assertRaises(SystemExit, self.parser.parse_args, []) def test_non_existing_config_dir(self): """ Run test mode and pass a non-existing configuration directory """ args = self.parser.parse_args(['test', 'foo']) - config = configparser.ConfigParser() - config.readfp(StringIO(cmd.DEFAULT_CONF)) + config = ConfigParser.ConfigParser() + config.readfp(cStringIO.StringIO(cmd.DEFAULT_CONF)) self.assertRaises(IOError, cmd.execute, args, config) def test_non_existing_config_file(self): @@ -38,8 +38,8 @@ class CmdTests(testtools.TestCase): Run test mode and pass a non-existing configuration file """ args = self.parser.parse_args(['test', 'non-existing.yaml']) - config = configparser.ConfigParser() - config.readfp(StringIO(cmd.DEFAULT_CONF)) + config = ConfigParser.ConfigParser() + config.readfp(cStringIO.StringIO(cmd.DEFAULT_CONF)) self.assertRaises(IOError, cmd.execute, args, config) def test_non_existing_job(self): @@ -52,8 +52,8 @@ class CmdTests(testtools.TestCase): 'cmd-001.yaml'), 'invalid']) args.output_dir = mock.MagicMock() - config = configparser.ConfigParser() - config.readfp(StringIO(cmd.DEFAULT_CONF)) + config = ConfigParser.ConfigParser() + config.readfp(cStringIO.StringIO(cmd.DEFAULT_CONF)) cmd.execute(args, config) # probably better to fail here def test_valid_job(self): @@ -65,8 +65,8 @@ class CmdTests(testtools.TestCase): 'cmd-001.yaml'), 'foo-job']) args.output_dir = mock.MagicMock() - config = configparser.ConfigParser() - config.readfp(StringIO(cmd.DEFAULT_CONF)) + config = ConfigParser.ConfigParser() + config.readfp(cStringIO.StringIO(cmd.DEFAULT_CONF)) cmd.execute(args, config) # probably better to fail here def test_console_output(self): @@ -74,14 +74,15 @@ class CmdTests(testtools.TestCase): Run test mode and verify that resulting XML gets sent to the console. """ - console_out = io.BytesIO() + console_out = cStringIO.StringIO() with mock.patch('sys.stdout', console_out): cmd.main(['test', os.path.join(self.fixtures_path, 'cmd-001.yaml')]) - xml_content = codecs.open(os.path.join(self.fixtures_path, - 'cmd-001.xml'), - 'r', 'utf-8').read() - self.assertEqual(console_out.getvalue().decode('utf-8'), xml_content) + xml_content = u"%s" % codecs.open(os.path.join(self.fixtures_path, + 'cmd-001.xml'), + 'r', + 'utf-8').read() + self.assertEqual(console_out.getvalue(), xml_content) def test_config_with_test(self): """