From a15c433bc88df67b95be8773c8d982d923ce2089 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 18 Jun 2012 17:33:47 -0700 Subject: [PATCH] Abate modules section. This is the first in a series of changes to alter the way modules are handled. This change removes the module section from yaml. Instead, all files in modules/ are imported, and a 'register' function is called if it exists. The register function can add (any number of) module classes to a module registry. The registry will also keep track of helper functions for use by other modules, and the register function can add those. This sets up the ability for a module other than the 'builders' module to register builders. Eventually, we should be able to move all OpenStack specific builders, publishers, etc outside of the base modules; instead we would just register an 'openstack_pep8' builder in a module of our own that the job filler would know to invoke. The API for modules is slightly changed, adding a root_xml method to handle the different project types, a handle_data method for modules that want to modify the yaml data structure before any XML is generated, and a data parameter is added to the generate_xml method. Ideally, we will migrate those modules that count on having a centrally stored data object to using the one passed into this method to allow maximum flexibility. I also envision some project-level yaml attributes to be moved closer to the handlers that use them. This change does inadvertently alter the XML produced. Here is the result of test.sh: http://paste.openstack.org/show/18585/ In all cases, those are inconsistencies in the YAML that are corrected by this change. Some jobs included an empty triggers vector (due to module trigger_none) while others did not, so there was no way to satisfy both behaviors. The added postbuilder section in the gerrit maven job is there because the job specified postbuilders, but did not include the postbuilder module. I believe the resulting XML is more desirable. Change-Id: Ib38222e6bfc9d5b55aa497669d7023c7aaf4b7bc --- modules/jenkins_jobs/files/jenkins_jobs.py | 81 ++++++++++------- .../files/modules/assignednode.py | 15 ++-- .../jenkins_jobs/files/modules/builders.py | 78 ++++++++++------- .../jenkins_jobs/files/modules/logrotate.py | 14 ++- .../{trigger_none.py => project_freestyle.py} | 27 ++++-- .../files/modules/project_maven.py | 21 +++-- .../jenkins_jobs/files/modules/properties.py | 13 ++- .../jenkins_jobs/files/modules/publishers.py | 46 +++++++--- modules/jenkins_jobs/files/modules/scm.py | 14 ++- .../files/modules/trigger_pollscm.py | 33 ------- .../files/modules/trigger_timed.py | 33 ------- .../{trigger_gerrit.py => triggers.py} | 86 +++++++++++++++---- modules/jenkins_jobs/files/modules/zuul.py | 20 +++-- .../projects/openstack/devstack-gate.yml | 8 -- .../files/projects/openstack/devstack.yml | 8 -- .../files/projects/openstack/gerrit.yml | 37 -------- .../openstack/openstack-ci-puppet.yml | 16 ---- .../files/projects/openstack/pypi-mirror.yml | 16 ---- .../files/projects/openstack/tempest.yml | 16 ---- .../files/projects/openstack/zuul.yml | 24 ------ .../files/projects/stackforge/ceilometer.yml | 16 ---- .../files/projects/stackforge/mraas.yml | 72 +++++++--------- .../files/templates/python_jobs.yml | 74 +--------------- 23 files changed, 316 insertions(+), 452 deletions(-) rename modules/jenkins_jobs/files/modules/{trigger_none.py => project_freestyle.py} (57%) delete mode 100644 modules/jenkins_jobs/files/modules/trigger_pollscm.py delete mode 100644 modules/jenkins_jobs/files/modules/trigger_timed.py rename modules/jenkins_jobs/files/modules/{trigger_gerrit.py => triggers.py} (56%) diff --git a/modules/jenkins_jobs/files/jenkins_jobs.py b/modules/jenkins_jobs/files/jenkins_jobs.py index e0ccc0da14..33ab59b657 100644 --- a/modules/jenkins_jobs/files/jenkins_jobs.py +++ b/modules/jenkins_jobs/files/jenkins_jobs.py @@ -25,6 +25,9 @@ import jenkins import ConfigParser from StringIO import StringIO import re +import pkgutil + +import modules class JenkinsJobsException(Exception): pass @@ -51,6 +54,7 @@ if not options.command == 'test': class YamlParser(object): def __init__(self, yfile): + self.registry = ModuleRegistry() self.data = yaml.load_all(yfile) self.it = self.data.__iter__() self.job_name = None @@ -79,10 +83,10 @@ class YamlParser(object): def get_next_xml(self): if not self.eof: if self.reading_template: - data = XmlParser(self.current_template) + data = XmlParser(self.current_template, self.registry) self.job_name = self.current_template['main']['name'] else: - data = XmlParser(self.current) + data = XmlParser(self.current, self.registry) self.job_name = self.current['main']['name'] self.seek_next_xml() return data @@ -108,37 +112,49 @@ class YamlParser(object): def get_name(self): return self.job_name -class XmlParser(object): - def __init__(self, data): - self.data = data - self.xml = XML.Element('project') +class ModuleRegistry(object): + # TODO: make this extensible + + def __init__(self): self.modules = [] - self._load_modules() + self.handlers = {} + + for importer, modname, ispkg in pkgutil.iter_modules(modules.__path__): + module = __import__('modules.'+modname, fromlist=['register']) + register = getattr(module, 'register', None) + if register: + register(self) + + def registerModule(self, mod): + self.modules.append(mod) + self.modules.sort(lambda a, b: cmp(a.sequence, b.sequence)) + + def registerHandler(self, category, name, method): + cat_dict = self.handlers.get(category, {}) + if not cat_dict: + self.handlers[category] = cat_dict + cat_dict[name] = method + + def getHandler(self, category, name): + return self.handlers[category][name] + +class XmlParser(object): + def __init__(self, data, registry): + self.data = data + self.registry = registry self._build() - def _load_modules(self): - for modulename in self.data['modules']: - full_modulename = 'modules.{name}'.format(name=modulename) - is_project = modulename.startswith('project_') - module = self._register_module(full_modulename, is_project) - if is_project: - self.xml = module.gen_xml(self.xml) - - def _register_module(self, modulename, skip=False): - class_and_alias = modulename.rsplit('.', 1)[1] - classname_split = class_and_alias.split(":") - classname = classname_split[0] - module = __import__(modulename.split(":")[0], fromlist=[classname]) - cla = getattr(module, classname) - if len(classname_split) > 1: - cla_instance = cla(self.data, classname_split[1]) - else: - cla_instance = cla(self.data) - if not skip: - self.modules.append(cla_instance) - return cla_instance - def _build(self): + for module in self.registry.modules: + if hasattr(module, 'root_xml'): + element = module.root_xml(self.data) + if element is not None: + self.xml = element + + for module in self.registry.modules: + if hasattr(module, 'handle_data'): + module.handle_data(self.data) + XML.SubElement(self.xml, 'actions') description = XML.SubElement(self.xml, 'description') description.text = "THIS JOB IS MANAGED BY PUPPET AND WILL BE OVERWRITTEN.\n\n\ @@ -158,11 +174,10 @@ In modules/jenkins_jobs" else: XML.SubElement(self.xml, 'concurrentBuild').text = 'false' XML.SubElement(self.xml, 'buildWrappers') - self._insert_modules() - def _insert_modules(self): - for module in self.modules: - module.gen_xml(self.xml) + for module in self.registry.modules: + if hasattr(module, 'gen_xml'): + module.gen_xml(self.xml, self.data) def md5(self): return hashlib.md5(self.output()).hexdigest() diff --git a/modules/jenkins_jobs/files/modules/assignednode.py b/modules/jenkins_jobs/files/modules/assignednode.py index 4748b2aae2..b40f22226e 100644 --- a/modules/jenkins_jobs/files/modules/assignednode.py +++ b/modules/jenkins_jobs/files/modules/assignednode.py @@ -20,12 +20,17 @@ import xml.etree.ElementTree as XML -class assignednode(object): - def __init__(self, data): - self.data = data - def gen_xml(self, xml_parent): - node = self.data['assignednode']['node'] +def register(registry): + mod = AssignedNode() + registry.registerModule(mod) + + +class AssignedNode(object): + sequence = 40 + + def gen_xml(self, xml_parent, data): + node = data['assignednode']['node'] XML.SubElement(xml_parent, 'assignedNode').text = node XML.SubElement(xml_parent, 'canRoam').text = 'false' diff --git a/modules/jenkins_jobs/files/modules/builders.py b/modules/jenkins_jobs/files/modules/builders.py index 5ba3d47710..bf808f67ec 100644 --- a/modules/jenkins_jobs/files/modules/builders.py +++ b/modules/jenkins_jobs/files/modules/builders.py @@ -21,60 +21,78 @@ import xml.etree.ElementTree as XML -class builders(object): - def __init__(self, data, alias='builders'): +def register(registry): + mod = Builders(registry) + registry.registerModule(mod) + + +class Builders(object): + sequence = 60 + + def __init__(self, registry): + self.registry = registry + for f in dir(self): + if not f.startswith('_builder_'): + continue + self.registry.registerHandler('builder', f[len('_builder_'):], + getattr(self, f)) + + def handle_data(self, data): self.data = data - self.alias = alias - def gen_xml(self, xml_parent): - builders = XML.SubElement(xml_parent, self.alias) - for builder in self.data[self.alias]: - if isinstance(builder, dict): - for key, value in builder.items(): - getattr(self, '_' + key)(builders, value) - else: - getattr(self, '_' + builder)(builders) + def gen_xml(self, xml_parent, data): + for alias in ['prebuilders', 'builders', 'postbuilders']: + if alias in data: + builders = XML.SubElement(xml_parent, alias) + for builder in data[alias]: + if isinstance(builder, dict): + for key, value in builder.items(): + func = self.registry.getHandler('builder', key) + func(builders, value) + else: + func = self.registry.getHandler('builder', builder) + func(builders) def _add_script(self, xml_parent, script): shell = XML.SubElement(xml_parent, 'hudson.tasks.Shell') XML.SubElement(shell, 'command').text = script - def _coverage(self, xml_parent): + def _builder_coverage(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/run-cover.sh') - def _docs(self, xml_parent): + def _builder_docs(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/run-docs.sh') - def _gerrit_git_prep(self, xml_parent): + def _builder_gerrit_git_prep(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/gerrit-git-prep.sh {site}'.format(site=self.data['main']['review_site'])) - def _maven_test(self, xml_parent): + def _builder_maven_test(self, xml_parent): self._add_script(xml_parent, 'mvn test') - def _maven_package(self, xml_parent): + def _builder_maven_package(self, xml_parent): self._add_script(xml_parent, 'mvn package') - def _gerrit_package(self, xml_parent): + def _builder_gerrit_package(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/package-gerrit.sh') - def _gerrit_preclean(self, xml_parent): + def _builder_gerrit_preclean(self, xml_parent): self._add_script(xml_parent, "#!/bin/bash -xe\n\ rm -fr ~/.m2\n\ rm -fr ~/.java\n\ ./tools/version.sh --release") - def _gerrit_postrun(self, xml_parent): + def _builder_gerrit_postrun(self, xml_parent): self._add_script(xml_parent, "./tools/version.sh --reset") - def _pep8(self, xml_parent): + def _builder_pep8(self, xml_parent): self._add_script(xml_parent, 'tox -v -epep8 | tee pep8.txt') - def _pyflakes(self, xml_parent): + def _builder_pyflakes(self, xml_parent): self._add_script(xml_parent, 'tox -v -epyflakes') - def _puppet_syntax(self, xml_parent): + def _builder_puppet_syntax(self, xml_parent): self._add_script(xml_parent, """ find . -iname *.pp | xargs puppet parser validate --modulepath=`pwd`/modules for f in `find . -iname *.erb` ; do @@ -82,10 +100,10 @@ for f in `find . -iname *.erb` ; do done """) - def _shell(self, xml_parent, data): + def _builder_shell(self, xml_parent, data): self._add_script(xml_parent, data) - def _trigger_builds(self, xml_parent, data): + def _builder_trigger_builds(self, xml_parent, data): tbuilder = XML.SubElement(xml_parent, 'hudson.plugins.parameterizedtrigger.TriggerBuilder') configs = XML.SubElement(tbuilder, 'configs') for project_def in data: @@ -107,23 +125,23 @@ done build_all_nodes_with_label = XML.SubElement(tconfig, 'buildAllNodesWithLabel') build_all_nodes_with_label.text = 'false' - def _python26(self, xml_parent): + def _builder_python26(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/run-tox.sh 26') - def _python27(self, xml_parent): + def _builder_python27(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/run-tox.sh 27') - def _python26_essex(self, xml_parent): + def _builder_python26_essex(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/run-tox.sh 26-essex') - def _python27_essex(self, xml_parent): + def _builder_python27_essex(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/run-tox.sh 27-essex') - def _tarball(self, xml_parent): + def _builder_tarball(self, xml_parent): self._add_script(xml_parent, '/usr/local/jenkins/slave_scripts/create-tarball.sh %s' % self.data['main']['project']) - def _ppa(self, xml_parent): + def _builder_ppa(self, xml_parent): self._add_script(xml_parent, 'rm -rf build dist.zip\n\ mkdir build') copy = XML.SubElement(xml_parent, 'hudson.plugins.copyartifact.CopyArtifact') diff --git a/modules/jenkins_jobs/files/modules/logrotate.py b/modules/jenkins_jobs/files/modules/logrotate.py index 5e7e5b7a47..9721b7a79d 100644 --- a/modules/jenkins_jobs/files/modules/logrotate.py +++ b/modules/jenkins_jobs/files/modules/logrotate.py @@ -23,11 +23,19 @@ import xml.etree.ElementTree as XML -class logrotate(object): - def __init__(self, data): + +def register(registry): + mod = LogRotate() + registry.registerModule(mod) + + +class LogRotate(object): + sequence = 10 + + def handle_data(self, data): self.data = data - def gen_xml(self, xml_parent): + def gen_xml(self, xml_parent, data): if self.data.has_key('logrotate'): lr_xml = XML.SubElement(xml_parent, 'logRotator') logrotate = self.data['logrotate'] diff --git a/modules/jenkins_jobs/files/modules/trigger_none.py b/modules/jenkins_jobs/files/modules/project_freestyle.py similarity index 57% rename from modules/jenkins_jobs/files/modules/trigger_none.py rename to modules/jenkins_jobs/files/modules/project_freestyle.py index b43b8b2f0c..72ce412145 100644 --- a/modules/jenkins_jobs/files/modules/trigger_none.py +++ b/modules/jenkins_jobs/files/modules/project_freestyle.py @@ -13,14 +13,27 @@ # License for the specific language governing permissions and limitations # under the License. -# Jenkins Job module for no triggers -# No additional YAML needed +# Jenkins Job module for maven projects +# To use you add the following into your YAML: +# maven: +# root_module: +# group_id: com.google.gerrit +# artifact_id: gerrit-parent +# goals: 'test' import xml.etree.ElementTree as XML -class trigger_none(object): - def __init__(self, data): - self.data = data - def gen_xml(self, xml_parent): - XML.SubElement(xml_parent, 'triggers', {'class':'vector'}) +def register(registry): + mod = Freestyle() + registry.registerModule(mod) + + +class Freestyle(object): + sequence = 0 + + def root_xml(self, data): + if 'maven' in data: + return None + xml_parent = XML.Element('project') + return xml_parent diff --git a/modules/jenkins_jobs/files/modules/project_maven.py b/modules/jenkins_jobs/files/modules/project_maven.py index 30cc4b86c7..0131c7d013 100644 --- a/modules/jenkins_jobs/files/modules/project_maven.py +++ b/modules/jenkins_jobs/files/modules/project_maven.py @@ -23,16 +23,23 @@ import xml.etree.ElementTree as XML -class project_maven(object): - def __init__(self, data): - self.data = data - def gen_xml(self, xml_parent): +def register(registry): + mod = Maven() + registry.registerModule(mod) + + +class Maven(object): + sequence = 0 + + def root_xml(self, data): + if 'maven' not in data: + return None xml_parent = XML.Element('maven2-moduleset') root_module = XML.SubElement(xml_parent, 'root_module') - XML.SubElement(root_module, 'groupId').text = self.data['maven']['root_module']['group_id'] - XML.SubElement(root_module, 'artifactId').text = self.data['maven']['root_module']['artifact_id'] - XML.SubElement(xml_parent, 'goals').text = self.data['maven']['goals'] + XML.SubElement(root_module, 'groupId').text = data['maven']['root_module']['group_id'] + XML.SubElement(root_module, 'artifactId').text = data['maven']['root_module']['artifact_id'] + XML.SubElement(xml_parent, 'goals').text = data['maven']['goals'] XML.SubElement(xml_parent, 'aggregatorStyleBuild').text = 'true' XML.SubElement(xml_parent, 'incrementalBuild').text = 'false' diff --git a/modules/jenkins_jobs/files/modules/properties.py b/modules/jenkins_jobs/files/modules/properties.py index 32b0c4acce..ead96e93d6 100644 --- a/modules/jenkins_jobs/files/modules/properties.py +++ b/modules/jenkins_jobs/files/modules/properties.py @@ -18,12 +18,19 @@ import xml.etree.ElementTree as XML -class properties(object): - def __init__(self, data): +def register(registry): + mod = Properties() + registry.registerModule(mod) + + +class Properties(object): + sequence = 20 + + def handle_data(self, data): self.data = data - def gen_xml(self, xml_parent): + def gen_xml(self, xml_parent, data): main = self.data['main'] properties = XML.SubElement(xml_parent, 'properties') if main.get('project'): diff --git a/modules/jenkins_jobs/files/modules/publishers.py b/modules/jenkins_jobs/files/modules/publishers.py index 62bd825eb1..528a27429b 100644 --- a/modules/jenkins_jobs/files/modules/publishers.py +++ b/modules/jenkins_jobs/files/modules/publishers.py @@ -18,21 +18,39 @@ import xml.etree.ElementTree as XML -class publishers(object): - def __init__(self, data): + +def register(registry): + mod = Publishers(registry) + registry.registerModule(mod) + + +class Publishers(object): + sequence = 70 + + def __init__(self, registry): + self.registry = registry + for f in dir(self): + if not f.startswith('_publisher_'): + continue + self.registry.registerHandler('publisher', f[len('_publisher_'):], + getattr(self, f)) + + def handle_data(self, data): self.data = data - def gen_xml(self, xml_parent): + def gen_xml(self, xml_parent, data): publishers = XML.SubElement(xml_parent, 'publishers') actions = self.data.get('post_build_actions', []) for action in actions: if isinstance(action, dict): for key, value in action.items(): - getattr(self, '_' + key)(publishers, value) + func = self.registry.getHandler('publisher', key) + func(publishers, value) else: - getattr(self, '_' + action)(publishers) + func = self.registry.getHandler('publisher', action) + func(publishers) - def _archive(self, xml_parent, data): + def _publisher_archive(self, xml_parent, data): archiver = XML.SubElement(xml_parent, 'hudson.tasks.ArtifactArchiver') artifacts = XML.SubElement(archiver, 'artifacts') artifacts.text = data['artifacts'] @@ -46,7 +64,7 @@ class publishers(object): else: latest.text = 'false' - def _trigger_parameterized_builds(self, xml_parent, data): + def _publisher_trigger_parameterized_builds(self, xml_parent, data): tbuilder = XML.SubElement(xml_parent, 'hudson.plugins.parameterizedtrigger.BuildTrigger') configs = XML.SubElement(tbuilder, 'configs') for project_def in data: @@ -66,7 +84,7 @@ class publishers(object): trigger_with_no_params = XML.SubElement(tconfig, 'triggerWithNoParameters') trigger_with_no_params.text = 'false' - def _coverage(self, xml_parent): + def _publisher_coverage(self, xml_parent): cobertura = XML.SubElement(xml_parent, 'hudson.plugins.cobertura.CoberturaPublisher') XML.SubElement(cobertura, 'coberturaReportFile').text = '**/coverage.xml' XML.SubElement(cobertura, 'onlyStable').text = 'false' @@ -116,7 +134,7 @@ class publishers(object): # This will upload everything under $workspace/base/source/dir to # docs.openstack.org $ftpdir/dest/dir exluding the excluded file type. - def _ftp(self, xml_parent, data): + def _publisher_ftp(self, xml_parent, data): """ Example XML: @@ -186,7 +204,7 @@ class publishers(object): # publisher: # results: 'nosetests.xml' - def _junit(self, xml_parent, data): + def _publisher_junit(self, xml_parent, data): junitresult = XML.SubElement(xml_parent, 'hudson.tasks.junit.JUnitResultArchiver') XML.SubElement(junitresult, 'testResults').text = data['results'] @@ -207,7 +225,7 @@ class publishers(object): XML.SubElement(tconfig, 'usePattern').text = 'false' XML.SubElement(tconfig, 'pattern') - def _pep8(self, xml_parent): + def _publisher_pep8(self, xml_parent): violations = XML.SubElement(xml_parent, 'hudson.plugins.violations.ViolationsPublisher') config = XML.SubElement(violations, 'config') suppressions = XML.SubElement(config, 'suppressions', {'class':'tree-set'}) @@ -249,7 +267,7 @@ class publishers(object): # Jenkins Job module for PPA publishers # No additional YAML needed - def _ppa(self, xml_parent): + def _publisher_ppa(self, xml_parent): archiver = XML.SubElement(xml_parent, 'hudson.tasks.ArtifactArchiver') XML.SubElement(archiver, 'artifacts').text = 'build/*.dsc,build/*.tar.gz,build/*.changes' XML.SubElement(archiver, 'latestOnly').text = 'false' @@ -259,7 +277,7 @@ class publishers(object): # publish: # site: 'glance.openstack.org' - def _tarball(self, xml_parent, data): + def _publisher_tarball(self, xml_parent, data): site = data['site'] archiver = XML.SubElement(xml_parent, 'hudson.tasks.ArtifactArchiver') XML.SubElement(archiver, 'artifacts').text = 'dist/*.tar.gz' @@ -279,7 +297,7 @@ class publishers(object): # warfile: 'gerrit-war/target/gerrit*.war' # target_path: 'tarballs/ci/' - def _war(self, xml_parent, data): + def _publisher_war(self, xml_parent, data): site = data['site'] archiver = XML.SubElement(xml_parent, 'hudson.tasks.ArtifactArchiver') XML.SubElement(archiver, 'artifacts').text = data['warfile'] diff --git a/modules/jenkins_jobs/files/modules/scm.py b/modules/jenkins_jobs/files/modules/scm.py index 44cf1e2311..1a4e039c92 100644 --- a/modules/jenkins_jobs/files/modules/scm.py +++ b/modules/jenkins_jobs/files/modules/scm.py @@ -22,11 +22,19 @@ import xml.etree.ElementTree as XML -class scm(object): - def __init__(self, data): + +def register(registry): + mod = SCM() + registry.registerModule(mod) + + +class SCM(object): + sequence = 30 + + def handle_data(self, data): self.data = data - def gen_xml(self, xml_parent): + def gen_xml(self, xml_parent, data): main = self.data['main'] scm_enabled = self.data['scm']['scm'] if scm_enabled == 'true': diff --git a/modules/jenkins_jobs/files/modules/trigger_pollscm.py b/modules/jenkins_jobs/files/modules/trigger_pollscm.py deleted file mode 100644 index 9d9eb69902..0000000000 --- a/modules/jenkins_jobs/files/modules/trigger_pollscm.py +++ /dev/null @@ -1,33 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Jenkins Job module for scm polling triggers -# To use add the following into your YAML: -# trigger: -# pollscm: '@midnight' -# or -# pollscm: '*/15 * * * *' - -import xml.etree.ElementTree as XML - -class trigger_pollscm(object): - def __init__(self, data): - self.data = data - - def gen_xml(self, xml_parent): - time = self.data['trigger']['pollscm'] - trigger = XML.SubElement(xml_parent, 'triggers', {'class':'vector'}) - scmtrig = XML.SubElement(trigger, 'hudson.triggers.SCMTrigger') - XML.SubElement(scmtrig, 'spec').text = time diff --git a/modules/jenkins_jobs/files/modules/trigger_timed.py b/modules/jenkins_jobs/files/modules/trigger_timed.py deleted file mode 100644 index 37686136d5..0000000000 --- a/modules/jenkins_jobs/files/modules/trigger_timed.py +++ /dev/null @@ -1,33 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Jenkins Job module for timed triggers -# To use add the following into your YAML: -# trigger: -# timed: '@midnight' -# or -# timed: '*/15 * * * *' - -import xml.etree.ElementTree as XML - -class trigger_timed(object): - def __init__(self, data): - self.data = data - - def gen_xml(self, xml_parent): - time = self.data['trigger']['timed'] - trigger = XML.SubElement(xml_parent, 'triggers', {'class':'vector'}) - scmtrig = XML.SubElement(trigger, 'hudson.triggers.TimerTrigger') - XML.SubElement(scmtrig, 'spec').text = time diff --git a/modules/jenkins_jobs/files/modules/trigger_gerrit.py b/modules/jenkins_jobs/files/modules/triggers.py similarity index 56% rename from modules/jenkins_jobs/files/modules/trigger_gerrit.py rename to modules/jenkins_jobs/files/modules/triggers.py index fa0f83290b..4658788133 100644 --- a/modules/jenkins_jobs/files/modules/trigger_gerrit.py +++ b/modules/jenkins_jobs/files/modules/triggers.py @@ -41,15 +41,43 @@ import xml.etree.ElementTree as XML -class trigger_gerrit(object): - def __init__(self, data): + +def register(registry): + mod = Triggers(registry) + registry.registerModule(mod) + + +class Triggers(object): + sequence = 50 + + def __init__(self, registry): + self.registry = registry + for f in dir(self): + if not f.startswith('_trigger_'): + continue + self.registry.registerHandler('trigger', f[len('_trigger_'):], + getattr(self, f)) + + def handle_data(self, data): self.data = data - def gen_xml(self, xml_parent): - trigger_data = self.data['trigger'] - projects = trigger_data['projects'] - trigger = XML.SubElement(xml_parent, 'triggers', {'class':'vector'}) - gtrig = XML.SubElement(trigger, 'com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger') + def gen_xml(self, xml_parent, data): + actions = self.data.get('triggers', []) + if not actions: + return + triggers = XML.SubElement(xml_parent, 'triggers', {'class':'vector'}) + for action in actions: + if isinstance(action, dict): + for key, value in action.items(): + func = self.registry.getHandler('trigger', key) + func(triggers, value) + else: + func = self.registry.getHandler('trigger', action) + func(triggers) + + def _trigger_gerrit(self, xml_parent, data): + projects = data['projects'] + gtrig = XML.SubElement(xml_parent, 'com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger') XML.SubElement(gtrig, 'spec') gprojects = XML.SubElement(gtrig, 'gerritProjects') for project in projects: @@ -62,18 +90,40 @@ class trigger_gerrit(object): XML.SubElement(gbranch, 'pattern').text = project['branchPattern'] XML.SubElement(gtrig, 'silentMode').text = 'false' XML.SubElement(gtrig, 'escapeQuotes').text = 'true' - XML.SubElement(gtrig, 'triggerOnPatchsetUploadedEvent').text = trigger_data['triggerOnPatchsetUploadedEvent'] - XML.SubElement(gtrig, 'triggerOnChangeMergedEvent').text = trigger_data['triggerOnChangeMergedEvent'] - XML.SubElement(gtrig, 'triggerOnCommentAddedEvent').text = trigger_data['triggerOnCommentAddedEvent'] - XML.SubElement(gtrig, 'triggerOnRefUpdatedEvent').text = trigger_data['triggerOnRefUpdatedEvent'] - if trigger_data.has_key('overrideVotes') and trigger_data['overrideVotes'] == 'true': - XML.SubElement(gtrig, 'gerritBuildSuccessfulVerifiedValue').text = str(trigger_data['gerritBuildSuccessfulVerifiedValue']) - XML.SubElement(gtrig, 'gerritBuildFailedVerifiedValue').text = str(trigger_data['gerritBuildFailedVerifiedValue']) - if trigger_data['triggerOnCommentAddedEvent'] == 'true': - XML.SubElement(gtrig, 'commentAddedTriggerApprovalCategory').text = trigger_data['triggerApprovalCategory'] - XML.SubElement(gtrig, 'commentAddedTriggerApprovalValue').text = str(trigger_data['triggerApprovalValue']) + XML.SubElement(gtrig, 'triggerOnPatchsetUploadedEvent').text = data['triggerOnPatchsetUploadedEvent'] + XML.SubElement(gtrig, 'triggerOnChangeMergedEvent').text = data['triggerOnChangeMergedEvent'] + XML.SubElement(gtrig, 'triggerOnCommentAddedEvent').text = data['triggerOnCommentAddedEvent'] + XML.SubElement(gtrig, 'triggerOnRefUpdatedEvent').text = data['triggerOnRefUpdatedEvent'] + if data.has_key('overrideVotes') and data['overrideVotes'] == 'true': + XML.SubElement(gtrig, 'gerritBuildSuccessfulVerifiedValue').text = str(data['gerritBuildSuccessfulVerifiedValue']) + XML.SubElement(gtrig, 'gerritBuildFailedVerifiedValue').text = str(data['gerritBuildFailedVerifiedValue']) + if data['triggerOnCommentAddedEvent'] == 'true': + XML.SubElement(gtrig, 'commentAddedTriggerApprovalCategory').text = data['triggerApprovalCategory'] + XML.SubElement(gtrig, 'commentAddedTriggerApprovalValue').text = str(data['triggerApprovalValue']) XML.SubElement(gtrig, 'buildStartMessage') - XML.SubElement(gtrig, 'buildFailureMessage').text = trigger_data['failureMessage'] + XML.SubElement(gtrig, 'buildFailureMessage').text = data['failureMessage'] XML.SubElement(gtrig, 'buildSuccessfulMessage') XML.SubElement(gtrig, 'buildUnstableMessage') XML.SubElement(gtrig, 'customUrl') + + # Jenkins Job module for scm polling triggers + # To use add the following into your YAML: + # trigger: + # pollscm: '@midnight' + # or + # pollscm: '*/15 * * * *' + + def _trigger_pollscm(self, xml_parent, data): + scmtrig = XML.SubElement(xml_parent, 'hudson.triggers.SCMTrigger') + XML.SubElement(scmtrig, 'spec').text = data + + # Jenkins Job module for timed triggers + # To use add the following into your YAML: + # trigger: + # timed: '@midnight' + # or + # timed: '*/15 * * * *' + + def _trigger_timed(self, xml_parent, data): + scmtrig = XML.SubElement(xml_parent, 'hudson.triggers.TimerTrigger') + XML.SubElement(scmtrig, 'spec').text = data diff --git a/modules/jenkins_jobs/files/modules/zuul.py b/modules/jenkins_jobs/files/modules/zuul.py index 20bc44daa9..54b59c3732 100644 --- a/modules/jenkins_jobs/files/modules/zuul.py +++ b/modules/jenkins_jobs/files/modules/zuul.py @@ -53,13 +53,16 @@ ZUUL_NOTIFICATIONS = [ 'protocol': 'HTTP'} ] -class zuul(object): - def __init__(self, data): - self.data = data - self._update() - def _update(self): - data = self.data +def register(registry): + mod = Zuul() + registry.registerModule(mod) + + +class Zuul(object): + sequence = 0 + + def handle_data(self, data): if ('zuul' not in data.get('triggers', []) and 'zuul_post' not in data.get('triggers', [])): return @@ -70,8 +73,7 @@ class zuul(object): data['notification_endpoints'].extend(ZUUL_NOTIFICATIONS) if 'zuul' in data.get('triggers', []): data['parameters'].extend(ZUUL_PARAMETERS) + data['triggers'].remove('zuul') if 'zuul_post' in data.get('triggers', []): data['parameters'].extend(ZUUL_POST_PARAMETERS) - - def gen_xml(self, xml_parent): - pass + data['triggers'].remove('zuul_post') diff --git a/modules/jenkins_jobs/files/projects/openstack/devstack-gate.yml b/modules/jenkins_jobs/files/projects/openstack/devstack-gate.yml index 6a1ed70939..1e7b7f7f01 100644 --- a/modules/jenkins_jobs/files/projects/openstack/devstack-gate.yml +++ b/modules/jenkins_jobs/files/projects/openstack/devstack-gate.yml @@ -1,13 +1,5 @@ --- # merge-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-devstack-gate-merge' review_site: 'review.openstack.org' diff --git a/modules/jenkins_jobs/files/projects/openstack/devstack.yml b/modules/jenkins_jobs/files/projects/openstack/devstack.yml index be0da442c3..5ddde2fe27 100644 --- a/modules/jenkins_jobs/files/projects/openstack/devstack.yml +++ b/modules/jenkins_jobs/files/projects/openstack/devstack.yml @@ -1,13 +1,5 @@ --- # merge-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-devstack-merge' review_site: 'review.openstack.org' diff --git a/modules/jenkins_jobs/files/projects/openstack/gerrit.yml b/modules/jenkins_jobs/files/projects/openstack/gerrit.yml index 742c3d19da..c27c76e0b8 100644 --- a/modules/jenkins_jobs/files/projects/openstack/gerrit.yml +++ b/modules/jenkins_jobs/files/projects/openstack/gerrit.yml @@ -1,13 +1,5 @@ --- # gate-gerrit-merge -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-gerrit-merge' github_org: 'openstack-ci' @@ -30,15 +22,6 @@ assignednode: --- # check-gerrit-unittests -modules: - - project_maven - - properties - - scm - - assignednode - - builders:prebuilders - - publishers - - zuul - main: name: 'check-gerrit-unittests' review_site: 'review.openstack.org' @@ -77,16 +60,6 @@ assignednode: --- # gate-gerrit-unittests -modules: - - project_maven - - properties - - scm - - assignednode - - builders:prebuilders - - builders:postbuilders - - publishers - - zuul - main: name: 'gate-gerrit-unittests' review_site: 'review.openstack.org' @@ -119,16 +92,6 @@ assignednode: --- # gerrit-package -modules: - - project_maven - - properties - - scm - - assignednode - - builders:prebuilders - - builders:postbuilders - - publishers - - zuul - main: name: 'gerrit-package' review_site: 'review.openstack.org' diff --git a/modules/jenkins_jobs/files/projects/openstack/openstack-ci-puppet.yml b/modules/jenkins_jobs/files/projects/openstack/openstack-ci-puppet.yml index 2e26a6815d..e05924b91a 100644 --- a/modules/jenkins_jobs/files/projects/openstack/openstack-ci-puppet.yml +++ b/modules/jenkins_jobs/files/projects/openstack/openstack-ci-puppet.yml @@ -1,13 +1,5 @@ --- # gate-ci-puppet-merge -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-ci-puppet-merge' review_site: 'review.openstack.org' @@ -29,14 +21,6 @@ assignednode: --- # gate-ci-puppet-syntax -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-ci-puppet-syntax' review_site: 'review.openstack.org' diff --git a/modules/jenkins_jobs/files/projects/openstack/pypi-mirror.yml b/modules/jenkins_jobs/files/projects/openstack/pypi-mirror.yml index 27a32e2c17..fc2c96c2e5 100644 --- a/modules/jenkins_jobs/files/projects/openstack/pypi-mirror.yml +++ b/modules/jenkins_jobs/files/projects/openstack/pypi-mirror.yml @@ -1,13 +1,5 @@ --- # pyflakes-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-pypi-mirror-pyflakes' review_site: 'review.openstack.org' @@ -30,14 +22,6 @@ assignednode: --- # merge-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-pypi-mirror-merge' review_site: 'review.openstack.org' diff --git a/modules/jenkins_jobs/files/projects/openstack/tempest.yml b/modules/jenkins_jobs/files/projects/openstack/tempest.yml index 60c8ab4974..f25be1a3b1 100644 --- a/modules/jenkins_jobs/files/projects/openstack/tempest.yml +++ b/modules/jenkins_jobs/files/projects/openstack/tempest.yml @@ -1,13 +1,5 @@ --- # pep8-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-tempest-pep8' review_site: 'review.openstack.org' @@ -33,14 +25,6 @@ assignednode: --- # merge-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-tempest-merge' review_site: 'review.openstack.org' diff --git a/modules/jenkins_jobs/files/projects/openstack/zuul.yml b/modules/jenkins_jobs/files/projects/openstack/zuul.yml index a82c9062e4..8106aedaa1 100644 --- a/modules/jenkins_jobs/files/projects/openstack/zuul.yml +++ b/modules/jenkins_jobs/files/projects/openstack/zuul.yml @@ -1,13 +1,5 @@ --- # pyflakes-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-zuul-pyflakes' review_site: 'review.openstack.org' @@ -30,14 +22,6 @@ assignednode: --- # pep8-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-zuul-pep8' review_site: 'review.openstack.org' @@ -63,14 +47,6 @@ assignednode: --- # merge-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-zuul-merge' review_site: 'review.openstack.org' diff --git a/modules/jenkins_jobs/files/projects/stackforge/ceilometer.yml b/modules/jenkins_jobs/files/projects/stackforge/ceilometer.yml index f3eb6bad26..bb9ba999be 100644 --- a/modules/jenkins_jobs/files/projects/stackforge/ceilometer.yml +++ b/modules/jenkins_jobs/files/projects/stackforge/ceilometer.yml @@ -12,14 +12,6 @@ values: --- # python26-essex-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-ceilometeer-python26-essex' review_site: 'review.stackforge.org' @@ -43,14 +35,6 @@ assignednode: --- # python27-essex-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-ceilometeer-python27-essex' review_site: 'review.stackforge.org' diff --git a/modules/jenkins_jobs/files/projects/stackforge/mraas.yml b/modules/jenkins_jobs/files/projects/stackforge/mraas.yml index 7922fdba58..872118828d 100644 --- a/modules/jenkins_jobs/files/projects/stackforge/mraas.yml +++ b/modules/jenkins_jobs/files/projects/stackforge/mraas.yml @@ -1,12 +1,4 @@ --- -modules: - - properties - - scm - - assignednode - - trigger_gerrit - - builders - - publishers - main: name: 'gate-MRaaS-merge' github_org: 'stackforge' @@ -14,19 +6,20 @@ main: project: 'MRaaS' authenticatedBuild: 'true' -trigger: - triggerOnPatchsetUploadedEvent: 'false' - triggerOnChangeMergedEvent: 'false' - triggerOnCommentAddedEvent: 'true' - triggerOnRefUpdatedEvent: 'false' - triggerApprovalCategory: 'APRV' - triggerApprovalValue: 1 - failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.' - projects: - - projectCompareType: 'PLAIN' - projectPattern: 'stackforge/MRaaS' - branchCompareType: 'ANT' - branchPattern: '**' +triggers: + - gerrit: + triggerOnPatchsetUploadedEvent: 'false' + triggerOnChangeMergedEvent: 'false' + triggerOnCommentAddedEvent: 'true' + triggerOnRefUpdatedEvent: 'false' + triggerApprovalCategory: 'APRV' + triggerApprovalValue: 1 + failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.' + projects: + - projectCompareType: 'PLAIN' + projectPattern: 'stackforge/MRaaS' + branchCompareType: 'ANT' + branchPattern: '**' builders: - gerrit_git_prep @@ -38,14 +31,6 @@ assignednode: node: 'oneiric' --- -modules: - - properties - - scm - - assignednode - - trigger_gerrit - - builders - - publishers - main: name: 'check-MRaaS-merge' github_org: 'stackforge' @@ -53,20 +38,21 @@ main: project: 'MRaaS' authenticatedBuild: 'true' -trigger: - triggerOnPatchsetUploadedEvent: 'true' - triggerOnChangeMergedEvent: 'false' - triggerOnCommentAddedEvent: 'false' - triggerOnRefUpdatedEvent: 'false' - overrideVotes: 'true' - gerritBuildSuccessfulVerifiedValue: 1 - gerritBuildFailedVerifiedValue: -1 - failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.' - projects: - - projectCompareType: 'PLAIN' - projectPattern: 'stackforge/MRaaS' - branchCompareType: 'ANT' - branchPattern: '**' +triggers: + - gerrit: + triggerOnPatchsetUploadedEvent: 'true' + triggerOnChangeMergedEvent: 'false' + triggerOnCommentAddedEvent: 'false' + triggerOnRefUpdatedEvent: 'false' + overrideVotes: 'true' + gerritBuildSuccessfulVerifiedValue: 1 + gerritBuildFailedVerifiedValue: -1 + failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.' + projects: + - projectCompareType: 'PLAIN' + projectPattern: 'stackforge/MRaaS' + branchCompareType: 'ANT' + branchPattern: '**' builders: - gerrit_git_prep diff --git a/modules/jenkins_jobs/files/templates/python_jobs.yml b/modules/jenkins_jobs/files/templates/python_jobs.yml index 2cce66080b..9ead3452f1 100644 --- a/modules/jenkins_jobs/files/templates/python_jobs.yml +++ b/modules/jenkins_jobs/files/templates/python_jobs.yml @@ -1,14 +1,5 @@ --- # coverage -modules: - - logrotate - - properties - - scm - - assignednode - - trigger_pollscm - - builders - - publishers - main: name: '@NAME@-coverage' review_site: '@REVIEW_SITE@' @@ -17,8 +8,8 @@ main: authenticatedBuild: 'false' disabled: @DISABLED@ -trigger: - pollscm: '*/15 * * * *' +triggers: + - pollscm: '*/15 * * * *' builders: - coverage @@ -40,14 +31,6 @@ logrotate: --- # pep8-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-@NAME@-pep8' review_site: '@REVIEW_SITE@' @@ -75,14 +58,6 @@ assignednode: --- # python26-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-@NAME@-python26' review_site: '@REVIEW_SITE@' @@ -112,14 +87,6 @@ assignednode: --- # python27-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-@NAME@-python27' review_site: '@REVIEW_SITE@' @@ -148,15 +115,6 @@ assignednode: --- # docs -modules: - - properties - - scm - - assignednode - - trigger_pollscm - - builders - - publishers - - zuul - main: name: '@NAME@-docs' review_site: '@REVIEW_SITE@' @@ -165,8 +123,8 @@ main: authenticatedBuild: 'false' disabled: @DISABLED@ -trigger: - pollscm: '*/15 * * * *' +triggers: + - pollscm: '*/15 * * * *' builders: - docs @@ -187,14 +145,6 @@ assignednode: --- # merge-gate -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: 'gate-@NAME@-merge' review_site: '@REVIEW_SITE@' @@ -218,14 +168,6 @@ assignednode: --- # ppa -modules: - - properties - - scm - - assignednode - - trigger_none - - builders - - publishers - main: name: '@NAME@-ppa' review_site: '@REVIEW_SITE@' @@ -248,14 +190,6 @@ assignednode: --- # tarball -modules: - - properties - - scm - - assignednode - - builders - - publishers - - zuul - main: name: '@NAME@-tarball' review_site: '@REVIEW_SITE@'