Remove YamlParser from jenkins_jobs.builder

The goal of this patch is simply to move some classes out of
jenkins_jobs.builder into more appropriately-named modules. This started with
simply moving YamlParser into jenkins_jobs.parser but led to other moves in
order to avoid cyclic imports since YamlParser uses other classes previously
defined in jenkins_jobs.builder.

That said, this patch doesn't intend to address all of the clutter in
jenkins_jobs.builder, mostly just what is necessary to get started working on
YamlParser independent of other classes in that module.

Change-Id: Ie88bf683e495033eb0b670fe29c256a70282735f
This commit is contained in:
Wayne 2015-04-22 09:18:45 -07:00
parent c29066210c
commit 4a8b93b8c2
10 changed files with 672 additions and 555 deletions

View File

@ -83,5 +83,5 @@ All modules and their associated components are registered in the
module registry. It can be accessed either from modules via the registry
field, or via the parser parameter of components.
.. autoclass:: jenkins_jobs.builder.ModuleRegistry
.. autoclass:: jenkins_jobs.registry.ModuleRegistry
:members:

View File

@ -26,40 +26,13 @@ import xml
from xml.dom import minidom
import jenkins
import re
import pkg_resources
from pprint import pformat
import logging
import copy
import itertools
import fnmatch
from string import Formatter
from jenkins_jobs.errors import JenkinsJobsException
import jenkins_jobs.local_yaml as local_yaml
from jenkins_jobs.constants import MAGIC_MANAGE_STRING
from jenkins_jobs.parser import YamlParser
logger = logging.getLogger(__name__)
MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"
class CustomFormatter(Formatter):
"""
Custom formatter to allow non-existing key references when formatting a
string
"""
def __init__(self, allow_empty=False):
super(CustomFormatter, self).__init__()
self.allow_empty = allow_empty
def get_value(self, key, args, kwargs):
try:
return Formatter.get_value(self, key, args, kwargs)
except KeyError:
if self.allow_empty:
logger.debug(
'Found uninitialized key %s, replaced with empty string',
key
)
return ''
raise
# Python 2.6's minidom toprettyxml produces broken output by adding extraneous
@ -99,526 +72,6 @@ if sys.version_info[:3] < (2, 7, 3) or xml.__name__ != 'xml':
minidom.Element.writexml = writexml
def deep_format(obj, paramdict, allow_empty=False):
"""Apply the paramdict via str.format() to all string objects found within
the supplied obj. Lists and dicts are traversed recursively."""
# YAML serialisation was originally used to achieve this, but that places
# limitations on the values in paramdict - the post-format result must
# still be valid YAML (so substituting-in a string containing quotes, for
# example, is problematic).
if hasattr(obj, 'format'):
try:
result = re.match('^{obj:(?P<key>\w+)}$', obj)
if result is not None:
ret = paramdict[result.group("key")]
else:
ret = CustomFormatter(allow_empty).format(obj, **paramdict)
except KeyError as exc:
missing_key = exc.message
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
missing_key, obj, pformat(paramdict))
raise JenkinsJobsException(desc)
elif isinstance(obj, list):
ret = []
for item in obj:
ret.append(deep_format(item, paramdict, allow_empty))
elif isinstance(obj, dict):
ret = {}
for item in obj:
try:
ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \
deep_format(obj[item], paramdict, allow_empty)
except KeyError as exc:
missing_key = exc.message
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
missing_key, obj, pformat(paramdict))
raise JenkinsJobsException(desc)
else:
ret = obj
return ret
def matches(what, glob_patterns):
"""
Checks if the given string, ``what``, matches any of the glob patterns in
the iterable, ``glob_patterns``
:arg str what: String that we want to test if it matches a pattern
:arg iterable glob_patterns: glob patterns to match (list, tuple, set,
etc.)
"""
return any(fnmatch.fnmatch(what, glob_pattern)
for glob_pattern in glob_patterns)
class YamlParser(object):
def __init__(self, config=None, plugins_info=None):
self.data = {}
self.jobs = []
self.xml_jobs = []
self.config = config
self.registry = ModuleRegistry(self.config, plugins_info)
self.path = ["."]
if self.config:
if config.has_section('job_builder') and \
config.has_option('job_builder', 'include_path'):
self.path = config.get('job_builder',
'include_path').split(':')
self.keep_desc = self.get_keep_desc()
def get_keep_desc(self):
keep_desc = False
if self.config and self.config.has_section('job_builder') and \
self.config.has_option('job_builder', 'keep_descriptions'):
keep_desc = self.config.getboolean('job_builder',
'keep_descriptions')
return keep_desc
def parse_fp(self, fp):
data = local_yaml.load(fp, search_path=self.path)
if data:
if not isinstance(data, list):
raise JenkinsJobsException(
"The topmost collection in file '{fname}' must be a list,"
" not a {cls}".format(fname=getattr(fp, 'name', fp),
cls=type(data)))
for item in data:
cls, dfn = next(iter(item.items()))
group = self.data.get(cls, {})
if len(item.items()) > 1:
n = None
for k, v in item.items():
if k == "name":
n = v
break
# Syntax error
raise JenkinsJobsException("Syntax error, for item "
"named '{0}'. Missing indent?"
.format(n))
name = dfn['name']
if name in group:
self._handle_dups("Duplicate entry found in '{0}: '{1}' "
"already defined".format(fp.name, name))
group[name] = dfn
self.data[cls] = group
def parse(self, fn):
with open(fn) as fp:
self.parse_fp(fp)
def _handle_dups(self, message):
if not (self.config and self.config.has_section('job_builder') and
self.config.getboolean('job_builder', 'allow_duplicates')):
logger.error(message)
raise JenkinsJobsException(message)
else:
logger.warn(message)
def getJob(self, name):
job = self.data.get('job', {}).get(name, None)
if not job:
return job
return self.applyDefaults(job)
def getJobGroup(self, name):
return self.data.get('job-group', {}).get(name, None)
def getJobTemplate(self, name):
job = self.data.get('job-template', {}).get(name, None)
if not job:
return job
return self.applyDefaults(job)
def applyDefaults(self, data, override_dict=None):
if override_dict is None:
override_dict = {}
whichdefaults = data.get('defaults', 'global')
defaults = copy.deepcopy(self.data.get('defaults',
{}).get(whichdefaults, {}))
if defaults == {} and whichdefaults != 'global':
raise JenkinsJobsException("Unknown defaults set: '{0}'"
.format(whichdefaults))
for key in override_dict.keys():
if key in defaults.keys():
defaults[key] = override_dict[key]
newdata = {}
newdata.update(defaults)
newdata.update(data)
return newdata
def formatDescription(self, job):
if self.keep_desc:
description = job.get("description", None)
else:
description = job.get("description", '')
if description is not None:
job["description"] = description + \
self.get_managed_string().lstrip()
def expandYaml(self, jobs_glob=None):
changed = True
while changed:
changed = False
for module in self.registry.modules:
if hasattr(module, 'handle_data'):
if module.handle_data(self):
changed = True
for job in self.data.get('job', {}).values():
if jobs_glob and not matches(job['name'], jobs_glob):
logger.debug("Ignoring job {0}".format(job['name']))
continue
logger.debug("Expanding job '{0}'".format(job['name']))
job = self.applyDefaults(job)
self.formatDescription(job)
self.jobs.append(job)
for project in self.data.get('project', {}).values():
logger.debug("Expanding project '{0}'".format(project['name']))
# use a set to check for duplicate job references in projects
seen = set()
for jobspec in project.get('jobs', []):
if isinstance(jobspec, dict):
# Singleton dict containing dict of job-specific params
jobname, jobparams = next(iter(jobspec.items()))
if not isinstance(jobparams, dict):
jobparams = {}
else:
jobname = jobspec
jobparams = {}
job = self.getJob(jobname)
if job:
# Just naming an existing defined job
if jobname in seen:
self._handle_dups("Duplicate job '{0}' specified "
"for project '{1}'".format(
jobname, project['name']))
seen.add(jobname)
continue
# see if it's a job group
group = self.getJobGroup(jobname)
if group:
for group_jobspec in group['jobs']:
if isinstance(group_jobspec, dict):
group_jobname, group_jobparams = \
next(iter(group_jobspec.items()))
if not isinstance(group_jobparams, dict):
group_jobparams = {}
else:
group_jobname = group_jobspec
group_jobparams = {}
job = self.getJob(group_jobname)
if job:
if group_jobname in seen:
self._handle_dups(
"Duplicate job '{0}' specified for "
"project '{1}'".format(group_jobname,
project['name']))
seen.add(group_jobname)
continue
template = self.getJobTemplate(group_jobname)
# Allow a group to override parameters set by a project
d = {}
d.update(project)
d.update(jobparams)
d.update(group)
d.update(group_jobparams)
# Except name, since the group's name is not useful
d['name'] = project['name']
if template:
self.expandYamlForTemplateJob(d, template,
jobs_glob)
continue
# see if it's a template
template = self.getJobTemplate(jobname)
if template:
d = {}
d.update(project)
d.update(jobparams)
self.expandYamlForTemplateJob(d, template, jobs_glob)
else:
raise JenkinsJobsException("Failed to find suitable "
"template named '{0}'"
.format(jobname))
# check for duplicate generated jobs
seen = set()
# walk the list in reverse so that last definition wins
for job in self.jobs[::-1]:
if job['name'] in seen:
self._handle_dups("Duplicate definitions for job '{0}' "
"specified".format(job['name']))
self.jobs.remove(job)
seen.add(job['name'])
def expandYamlForTemplateJob(self, project, template, jobs_glob=None):
dimensions = []
template_name = template['name']
# reject keys that are not useful during yaml expansion
for k in ['jobs']:
project.pop(k)
for (k, v) in project.items():
tmpk = '{{{0}}}'.format(k)
if tmpk not in template_name:
logger.debug("Variable %s not in name %s, rejecting from job"
" matrix expansion.", tmpk, template_name)
continue
if type(v) == list:
dimensions.append(zip([k] * len(v), v))
# XXX somewhat hackish to ensure we actually have a single
# pass through the loop
if len(dimensions) == 0:
dimensions = [(("", ""),)]
for values in itertools.product(*dimensions):
params = copy.deepcopy(project)
params = self.applyDefaults(params, template)
expanded_values = {}
for (k, v) in values:
if isinstance(v, dict):
inner_key = next(iter(v))
expanded_values[k] = inner_key
expanded_values.update(v[inner_key])
else:
expanded_values[k] = v
params.update(expanded_values)
params = deep_format(params, params)
allow_empty_variables = self.config \
and self.config.has_section('job_builder') \
and self.config.has_option(
'job_builder', 'allow_empty_variables') \
and self.config.getboolean(
'job_builder', 'allow_empty_variables')
expanded = deep_format(template, params, allow_empty_variables)
job_name = expanded.get('name')
if jobs_glob and not matches(job_name, jobs_glob):
continue
self.formatDescription(expanded)
self.jobs.append(expanded)
def get_managed_string(self):
# The \n\n is not hard coded, because they get stripped if the
# project does not otherwise have a description.
return "\n\n" + MAGIC_MANAGE_STRING
def generateXML(self):
for job in self.jobs:
self.xml_jobs.append(self.getXMLForJob(job))
def getXMLForJob(self, data):
kind = data.get('project-type', 'freestyle')
for ep in pkg_resources.iter_entry_points(
group='jenkins_jobs.projects', name=kind):
Mod = ep.load()
mod = Mod(self.registry)
xml = mod.root_xml(data)
self.gen_xml(xml, data)
job = XmlJob(xml, data['name'])
return job
def gen_xml(self, xml, data):
for module in self.registry.modules:
if hasattr(module, 'gen_xml'):
module.gen_xml(self, xml, data)
class ModuleRegistry(object):
entry_points_cache = {}
def __init__(self, config, plugins_list=None):
self.modules = []
self.modules_by_component_type = {}
self.handlers = {}
self.global_config = config
if plugins_list is None:
self.plugins_dict = {}
else:
self.plugins_dict = self._get_plugins_info_dict(plugins_list)
for entrypoint in pkg_resources.iter_entry_points(
group='jenkins_jobs.modules'):
Mod = entrypoint.load()
mod = Mod(self)
self.modules.append(mod)
self.modules.sort(key=operator.attrgetter('sequence'))
if mod.component_type is not None:
self.modules_by_component_type[mod.component_type] = mod
@staticmethod
def _get_plugins_info_dict(plugins_list):
def mutate_plugin_info(plugin_info):
"""
We perform mutations on a single member of plugin_info here, then
return a dictionary with the longName and shortName of the plugin
mapped to its plugin info dictionary.
"""
version = plugin_info.get('version', '0')
plugin_info['version'] = re.sub(r'(.*)-(?:SNAPSHOT|BETA)',
r'\g<1>.preview', version)
aliases = []
for key in ['longName', 'shortName']:
value = plugin_info.get(key, None)
if value is not None:
aliases.append(value)
plugin_info_dict = {}
for name in aliases:
plugin_info_dict[name] = plugin_info
return plugin_info_dict
list_of_dicts = [mutate_plugin_info(v) for v in plugins_list]
plugins_info_dict = {}
for d in list_of_dicts:
plugins_info_dict.update(d)
return plugins_info_dict
def get_plugin_info(self, plugin_name):
""" This method is intended to provide information about plugins within
a given module's implementation of Base.gen_xml. The return value is a
dictionary with data obtained directly from a running Jenkins instance.
This allows module authors to differentiate generated XML output based
on information such as specific plugin versions.
:arg string plugin_name: Either the shortName or longName of a plugin
as see in a query that looks like:
``http://<jenkins-hostname>/pluginManager/api/json?pretty&depth=2``
During a 'test' run, it is possible to override JJB's query to a live
Jenkins instance by passing it a path to a file containing a YAML list
of dictionaries that mimics the plugin properties you want your test
output to reflect::
jenkins-jobs test -p /path/to/plugins-info.yaml
Below is example YAML that might be included in
/path/to/plugins-info.yaml.
.. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml
"""
return self.plugins_dict.get(plugin_name, {})
def registerHandler(self, category, name, method):
cat_dict = self.handlers.get(category, {})
if not cat_dict:
self.handlers[category] = cat_dict
cat_dict[name] = method
def getHandler(self, category, name):
return self.handlers[category][name]
def dispatch(self, component_type,
parser, xml_parent,
component, template_data={}):
"""This is a method that you can call from your implementation of
Base.gen_xml or component. It allows modules to define a type
of component, and benefit from extensibility via Python
entry points and Jenkins Job Builder :ref:`Macros <macro>`.
:arg string component_type: the name of the component
(e.g., `builder`)
:arg YAMLParser parser: the global YAML Parser
:arg Element xml_parent: the parent XML element
:arg dict template_data: values that should be interpolated into
the component definition
See :py:class:`jenkins_jobs.modules.base.Base` for how to register
components of a module.
See the Publishers module for a simple example of how to use
this method.
"""
if component_type not in self.modules_by_component_type:
raise JenkinsJobsException("Unknown component type: "
"'{0}'.".format(component_type))
component_list_type = self.modules_by_component_type[component_type] \
.component_list_type
if isinstance(component, dict):
# The component is a singleton dictionary of name: dict(args)
name, component_data = next(iter(component.items()))
if template_data:
# Template data contains values that should be interpolated
# into the component definition
s = yaml.dump(component_data, default_flow_style=False)
allow_empty_variables = self.global_config \
and self.global_config.has_section('job_builder') \
and self.global_config.has_option(
'job_builder', 'allow_empty_variables') \
and self.global_config.getboolean(
'job_builder', 'allow_empty_variables')
s = CustomFormatter(
allow_empty_variables).format(s, **template_data)
component_data = yaml.load(s)
else:
# The component is a simple string name, eg "run-tests"
name = component
component_data = {}
# Look for a component function defined in an entry point
eps = ModuleRegistry.entry_points_cache.get(component_list_type)
if eps is None:
module_eps = list(pkg_resources.iter_entry_points(
group='jenkins_jobs.{0}'.format(component_list_type)))
eps = {}
for module_ep in module_eps:
if module_ep.name in eps:
raise JenkinsJobsException(
"Duplicate entry point found for component type: "
"'{0}', '{0}',"
"name: '{1}'".format(component_type, name))
eps[module_ep.name] = module_ep
ModuleRegistry.entry_points_cache[component_list_type] = eps
logger.debug("Cached entry point group %s = %s",
component_list_type, eps)
if name in eps:
func = eps[name].load()
func(parser, xml_parent, component_data)
else:
# Otherwise, see if it's defined as a macro
component = parser.data.get(component_type, {}).get(name)
if component:
for b in component[component_list_type]:
# Pass component_data in as template data to this function
# so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn.
self.dispatch(component_type,
parser, xml_parent, b, component_data)
else:
raise JenkinsJobsException("Unknown entry point or macro '{0}'"
" for component type: '{1}'.".
format(name, component_type))
class XmlJob(object):
def __init__(self, xml, name):
self.xml = xml
self.name = name
def md5(self):
return hashlib.md5(self.output()).hexdigest()
def output(self):
out = minidom.parseString(XML.tostring(self.xml, encoding='UTF-8'))
return out.toprettyxml(indent=' ', encoding='utf-8')
class CacheStorage(object):
# ensure each instance of the class has a reference to the required
# modules so that they are available to be used when the destructor

18
jenkins_jobs/constants.py Normal file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env python
# Copyright (C) 2015 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Manage JJB Constants
MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"

86
jenkins_jobs/formatter.py Normal file
View File

@ -0,0 +1,86 @@
#!/usr/bin/env python
# Copyright (C) 2015 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Manage interpolation of JJB variables into template strings.
import logging
from pprint import pformat
import re
from string import Formatter
from jenkins_jobs.errors import JenkinsJobsException
logger = logging.getLogger(__name__)
def deep_format(obj, paramdict, allow_empty=False):
"""Apply the paramdict via str.format() to all string objects found within
the supplied obj. Lists and dicts are traversed recursively."""
# YAML serialisation was originally used to achieve this, but that places
# limitations on the values in paramdict - the post-format result must
# still be valid YAML (so substituting-in a string containing quotes, for
# example, is problematic).
if hasattr(obj, 'format'):
try:
result = re.match('^{obj:(?P<key>\w+)}$', obj)
if result is not None:
ret = paramdict[result.group("key")]
else:
ret = CustomFormatter(allow_empty).format(obj, **paramdict)
except KeyError as exc:
missing_key = exc.message
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
missing_key, obj, pformat(paramdict))
raise JenkinsJobsException(desc)
elif isinstance(obj, list):
ret = []
for item in obj:
ret.append(deep_format(item, paramdict, allow_empty))
elif isinstance(obj, dict):
ret = {}
for item in obj:
try:
ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \
deep_format(obj[item], paramdict, allow_empty)
except KeyError as exc:
missing_key = exc.message
desc = "%s parameter missing to format %s\nGiven:\n%s" % (
missing_key, obj, pformat(paramdict))
raise JenkinsJobsException(desc)
else:
ret = obj
return ret
class CustomFormatter(Formatter):
"""
Custom formatter to allow non-existing key references when formatting a
string
"""
def __init__(self, allow_empty=False):
super(CustomFormatter, self).__init__()
self.allow_empty = allow_empty
def get_value(self, key, args, kwargs):
try:
return Formatter.get_value(self, key, args, kwargs)
except KeyError:
if self.allow_empty:
logger.debug(
'Found uninitialized key %s, replaced with empty string',
key
)
return ''
raise

322
jenkins_jobs/parser.py Normal file
View File

@ -0,0 +1,322 @@
#!/usr/bin/env python
# Copyright (C) 2015 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Manage JJB yaml feature implementation
import copy
import fnmatch
import itertools
import logging
import pkg_resources
import jenkins_jobs.local_yaml as local_yaml
from jenkins_jobs.constants import MAGIC_MANAGE_STRING
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.registry import ModuleRegistry
from jenkins_jobs.formatter import deep_format
from jenkins_jobs.xml_config import XmlJob
logger = logging.getLogger(__name__)
def matches(what, glob_patterns):
"""
Checks if the given string, ``what``, matches any of the glob patterns in
the iterable, ``glob_patterns``
:arg str what: String that we want to test if it matches a pattern
:arg iterable glob_patterns: glob patterns to match (list, tuple, set,
etc.)
"""
return any(fnmatch.fnmatch(what, glob_pattern)
for glob_pattern in glob_patterns)
class YamlParser(object):
def __init__(self, config=None, plugins_info=None):
self.data = {}
self.jobs = []
self.xml_jobs = []
self.config = config
self.registry = ModuleRegistry(self.config, plugins_info)
self.path = ["."]
if self.config:
if config.has_section('job_builder') and \
config.has_option('job_builder', 'include_path'):
self.path = config.get('job_builder',
'include_path').split(':')
self.keep_desc = self.get_keep_desc()
def get_keep_desc(self):
keep_desc = False
if self.config and self.config.has_section('job_builder') and \
self.config.has_option('job_builder', 'keep_descriptions'):
keep_desc = self.config.getboolean('job_builder',
'keep_descriptions')
return keep_desc
def parse_fp(self, fp):
data = local_yaml.load(fp, search_path=self.path)
if data:
if not isinstance(data, list):
raise JenkinsJobsException(
"The topmost collection in file '{fname}' must be a list,"
" not a {cls}".format(fname=getattr(fp, 'name', fp),
cls=type(data)))
for item in data:
cls, dfn = next(iter(item.items()))
group = self.data.get(cls, {})
if len(item.items()) > 1:
n = None
for k, v in item.items():
if k == "name":
n = v
break
# Syntax error
raise JenkinsJobsException("Syntax error, for item "
"named '{0}'. Missing indent?"
.format(n))
name = dfn['name']
if name in group:
self._handle_dups("Duplicate entry found in '{0}: '{1}' "
"already defined".format(fp.name, name))
group[name] = dfn
self.data[cls] = group
def parse(self, fn):
with open(fn) as fp:
self.parse_fp(fp)
def _handle_dups(self, message):
if not (self.config and self.config.has_section('job_builder') and
self.config.getboolean('job_builder', 'allow_duplicates')):
logger.error(message)
raise JenkinsJobsException(message)
else:
logger.warn(message)
def getJob(self, name):
job = self.data.get('job', {}).get(name, None)
if not job:
return job
return self.applyDefaults(job)
def getJobGroup(self, name):
return self.data.get('job-group', {}).get(name, None)
def getJobTemplate(self, name):
job = self.data.get('job-template', {}).get(name, None)
if not job:
return job
return self.applyDefaults(job)
def applyDefaults(self, data, override_dict=None):
if override_dict is None:
override_dict = {}
whichdefaults = data.get('defaults', 'global')
defaults = copy.deepcopy(self.data.get('defaults',
{}).get(whichdefaults, {}))
if defaults == {} and whichdefaults != 'global':
raise JenkinsJobsException("Unknown defaults set: '{0}'"
.format(whichdefaults))
for key in override_dict.keys():
if key in defaults.keys():
defaults[key] = override_dict[key]
newdata = {}
newdata.update(defaults)
newdata.update(data)
return newdata
def formatDescription(self, job):
if self.keep_desc:
description = job.get("description", None)
else:
description = job.get("description", '')
if description is not None:
job["description"] = description + \
self.get_managed_string().lstrip()
def expandYaml(self, jobs_glob=None):
changed = True
while changed:
changed = False
for module in self.registry.modules:
if hasattr(module, 'handle_data'):
if module.handle_data(self):
changed = True
for job in self.data.get('job', {}).values():
if jobs_glob and not matches(job['name'], jobs_glob):
logger.debug("Ignoring job {0}".format(job['name']))
continue
logger.debug("Expanding job '{0}'".format(job['name']))
job = self.applyDefaults(job)
self.formatDescription(job)
self.jobs.append(job)
for project in self.data.get('project', {}).values():
logger.debug("Expanding project '{0}'".format(project['name']))
# use a set to check for duplicate job references in projects
seen = set()
for jobspec in project.get('jobs', []):
if isinstance(jobspec, dict):
# Singleton dict containing dict of job-specific params
jobname, jobparams = next(iter(jobspec.items()))
if not isinstance(jobparams, dict):
jobparams = {}
else:
jobname = jobspec
jobparams = {}
job = self.getJob(jobname)
if job:
# Just naming an existing defined job
if jobname in seen:
self._handle_dups("Duplicate job '{0}' specified "
"for project '{1}'".format(
jobname, project['name']))
seen.add(jobname)
continue
# see if it's a job group
group = self.getJobGroup(jobname)
if group:
for group_jobspec in group['jobs']:
if isinstance(group_jobspec, dict):
group_jobname, group_jobparams = \
next(iter(group_jobspec.items()))
if not isinstance(group_jobparams, dict):
group_jobparams = {}
else:
group_jobname = group_jobspec
group_jobparams = {}
job = self.getJob(group_jobname)
if job:
if group_jobname in seen:
self._handle_dups(
"Duplicate job '{0}' specified for "
"project '{1}'".format(group_jobname,
project['name']))
seen.add(group_jobname)
continue
template = self.getJobTemplate(group_jobname)
# Allow a group to override parameters set by a project
d = {}
d.update(project)
d.update(jobparams)
d.update(group)
d.update(group_jobparams)
# Except name, since the group's name is not useful
d['name'] = project['name']
if template:
self.expandYamlForTemplateJob(d, template,
jobs_glob)
continue
# see if it's a template
template = self.getJobTemplate(jobname)
if template:
d = {}
d.update(project)
d.update(jobparams)
self.expandYamlForTemplateJob(d, template, jobs_glob)
else:
raise JenkinsJobsException("Failed to find suitable "
"template named '{0}'"
.format(jobname))
# check for duplicate generated jobs
seen = set()
# walk the list in reverse so that last definition wins
for job in self.jobs[::-1]:
if job['name'] in seen:
self._handle_dups("Duplicate definitions for job '{0}' "
"specified".format(job['name']))
self.jobs.remove(job)
seen.add(job['name'])
def expandYamlForTemplateJob(self, project, template, jobs_glob=None):
dimensions = []
template_name = template['name']
# reject keys that are not useful during yaml expansion
for k in ['jobs']:
project.pop(k)
for (k, v) in project.items():
tmpk = '{{{0}}}'.format(k)
if tmpk not in template_name:
logger.debug("Variable %s not in name %s, rejecting from job"
" matrix expansion.", tmpk, template_name)
continue
if type(v) == list:
dimensions.append(zip([k] * len(v), v))
# XXX somewhat hackish to ensure we actually have a single
# pass through the loop
if len(dimensions) == 0:
dimensions = [(("", ""),)]
for values in itertools.product(*dimensions):
params = copy.deepcopy(project)
params = self.applyDefaults(params, template)
expanded_values = {}
for (k, v) in values:
if isinstance(v, dict):
inner_key = next(iter(v))
expanded_values[k] = inner_key
expanded_values.update(v[inner_key])
else:
expanded_values[k] = v
params.update(expanded_values)
params = deep_format(params, params)
allow_empty_variables = self.config \
and self.config.has_section('job_builder') \
and self.config.has_option(
'job_builder', 'allow_empty_variables') \
and self.config.getboolean(
'job_builder', 'allow_empty_variables')
expanded = deep_format(template, params, allow_empty_variables)
job_name = expanded.get('name')
if jobs_glob and not matches(job_name, jobs_glob):
continue
self.formatDescription(expanded)
self.jobs.append(expanded)
def get_managed_string(self):
# The \n\n is not hard coded, because they get stripped if the
# project does not otherwise have a description.
return "\n\n" + MAGIC_MANAGE_STRING
def generateXML(self):
for job in self.jobs:
self.xml_jobs.append(self.getXMLForJob(job))
def getXMLForJob(self, data):
kind = data.get('project-type', 'freestyle')
for ep in pkg_resources.iter_entry_points(
group='jenkins_jobs.projects', name=kind):
Mod = ep.load()
mod = Mod(self.registry)
xml = mod.root_xml(data)
self.gen_xml(xml, data)
job = XmlJob(xml, data['name'])
return job
def gen_xml(self, xml, data):
for module in self.registry.modules:
if hasattr(module, 'gen_xml'):
module.gen_xml(self, xml, data)

204
jenkins_jobs/registry.py Normal file
View File

@ -0,0 +1,204 @@
#!/usr/bin/env python
# Copyright (C) 2015 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Manage Jenkins plugin module registry.
import logging
import operator
import pkg_resources
import re
import yaml
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.formatter import CustomFormatter
logger = logging.getLogger(__name__)
class ModuleRegistry(object):
entry_points_cache = {}
def __init__(self, config, plugins_list=None):
self.modules = []
self.modules_by_component_type = {}
self.handlers = {}
self.global_config = config
if plugins_list is None:
self.plugins_dict = {}
else:
self.plugins_dict = self._get_plugins_info_dict(plugins_list)
for entrypoint in pkg_resources.iter_entry_points(
group='jenkins_jobs.modules'):
Mod = entrypoint.load()
mod = Mod(self)
self.modules.append(mod)
self.modules.sort(key=operator.attrgetter('sequence'))
if mod.component_type is not None:
self.modules_by_component_type[mod.component_type] = mod
@staticmethod
def _get_plugins_info_dict(plugins_list):
def mutate_plugin_info(plugin_info):
"""
We perform mutations on a single member of plugin_info here, then
return a dictionary with the longName and shortName of the plugin
mapped to its plugin info dictionary.
"""
version = plugin_info.get('version', '0')
plugin_info['version'] = re.sub(r'(.*)-(?:SNAPSHOT|BETA)',
r'\g<1>.preview', version)
aliases = []
for key in ['longName', 'shortName']:
value = plugin_info.get(key, None)
if value is not None:
aliases.append(value)
plugin_info_dict = {}
for name in aliases:
plugin_info_dict[name] = plugin_info
return plugin_info_dict
list_of_dicts = [mutate_plugin_info(v) for v in plugins_list]
plugins_info_dict = {}
for d in list_of_dicts:
plugins_info_dict.update(d)
return plugins_info_dict
def get_plugin_info(self, plugin_name):
""" This method is intended to provide information about plugins within
a given module's implementation of Base.gen_xml. The return value is a
dictionary with data obtained directly from a running Jenkins instance.
This allows module authors to differentiate generated XML output based
on information such as specific plugin versions.
:arg string plugin_name: Either the shortName or longName of a plugin
as see in a query that looks like:
``http://<jenkins-hostname>/pluginManager/api/json?pretty&depth=2``
During a 'test' run, it is possible to override JJB's query to a live
Jenkins instance by passing it a path to a file containing a YAML list
of dictionaries that mimics the plugin properties you want your test
output to reflect::
jenkins-jobs test -p /path/to/plugins-info.yaml
Below is example YAML that might be included in
/path/to/plugins-info.yaml.
.. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml
"""
return self.plugins_dict.get(plugin_name, {})
def registerHandler(self, category, name, method):
cat_dict = self.handlers.get(category, {})
if not cat_dict:
self.handlers[category] = cat_dict
cat_dict[name] = method
def getHandler(self, category, name):
return self.handlers[category][name]
def dispatch(self, component_type,
parser, xml_parent,
component, template_data={}):
"""This is a method that you can call from your implementation of
Base.gen_xml or component. It allows modules to define a type
of component, and benefit from extensibility via Python
entry points and Jenkins Job Builder :ref:`Macros <macro>`.
:arg string component_type: the name of the component
(e.g., `builder`)
:arg YAMLParser parser: the global YAML Parser
:arg Element xml_parent: the parent XML element
:arg dict template_data: values that should be interpolated into
the component definition
See :py:class:`jenkins_jobs.modules.base.Base` for how to register
components of a module.
See the Publishers module for a simple example of how to use
this method.
"""
if component_type not in self.modules_by_component_type:
raise JenkinsJobsException("Unknown component type: "
"'{0}'.".format(component_type))
component_list_type = self.modules_by_component_type[component_type] \
.component_list_type
if isinstance(component, dict):
# The component is a singleton dictionary of name: dict(args)
name, component_data = next(iter(component.items()))
if template_data:
# Template data contains values that should be interpolated
# into the component definition
s = yaml.dump(component_data, default_flow_style=False)
allow_empty_variables = self.global_config \
and self.global_config.has_section('job_builder') \
and self.global_config.has_option(
'job_builder', 'allow_empty_variables') \
and self.global_config.getboolean(
'job_builder', 'allow_empty_variables')
s = CustomFormatter(
allow_empty_variables).format(s, **template_data)
component_data = yaml.load(s)
else:
# The component is a simple string name, eg "run-tests"
name = component
component_data = {}
# Look for a component function defined in an entry point
eps = ModuleRegistry.entry_points_cache.get(component_list_type)
if eps is None:
module_eps = list(pkg_resources.iter_entry_points(
group='jenkins_jobs.{0}'.format(component_list_type)))
eps = {}
for module_ep in module_eps:
if module_ep.name in eps:
raise JenkinsJobsException(
"Duplicate entry point found for component type: "
"'{0}', '{0}',"
"name: '{1}'".format(component_type, name))
eps[module_ep.name] = module_ep
ModuleRegistry.entry_points_cache[component_list_type] = eps
logger.debug("Cached entry point group %s = %s",
component_list_type, eps)
if name in eps:
func = eps[name].load()
func(parser, xml_parent, component_data)
else:
# Otherwise, see if it's defined as a macro
component = parser.data.get(component_type, {}).get(name)
if component:
for b in component[component_list_type]:
# Pass component_data in as template data to this function
# so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn.
self.dispatch(component_type,
parser, xml_parent, b, component_data)
else:
raise JenkinsJobsException("Unknown entry point or macro '{0}'"
" for component type: '{1}'.".
format(name, component_type))

View File

@ -0,0 +1,33 @@
#!/usr/bin/env python
# Copyright (C) 2015 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Manage Jenkins XML config file output.
import hashlib
from xml.dom import minidom
import xml.etree.ElementTree as XML
class XmlJob(object):
def __init__(self, xml, name):
self.xml = xml
self.name = name
def md5(self):
return hashlib.md5(self.output()).hexdigest()
def output(self):
out = minidom.parseString(XML.tostring(self.xml, encoding='UTF-8'))
return out.toprettyxml(indent=' ', encoding='utf-8')

View File

@ -37,7 +37,8 @@ try:
except ImportError:
import mock # noqa
import jenkins_jobs.local_yaml as yaml
from jenkins_jobs.builder import XmlJob, YamlParser
from jenkins_jobs.parser import YamlParser
from jenkins_jobs.xml_config import XmlJob
from jenkins_jobs.modules import (project_flow,
project_matrix,
project_maven,

View File

@ -190,7 +190,7 @@ class TestTests(CmdTestsBase):
"http://test-jenkins.with.non.default.url:8080/")
@mock.patch('jenkins_jobs.builder.YamlParser.generateXML')
@mock.patch('jenkins_jobs.builder.ModuleRegistry')
@mock.patch('jenkins_jobs.parser.ModuleRegistry')
def test_plugins_info_stub_option(self, registry_mock, generateXML_mock):
"""
Test handling of plugins_info stub option.
@ -214,7 +214,7 @@ class TestTests(CmdTestsBase):
registry_mock.assert_called_with(self.config, plugins_info_list)
@mock.patch('jenkins_jobs.builder.YamlParser.generateXML')
@mock.patch('jenkins_jobs.builder.ModuleRegistry')
@mock.patch('jenkins_jobs.parser.ModuleRegistry')
def test_bogus_plugins_info_stub_option(self, registry_mock,
generateXML_mock):
"""

View File

@ -5,7 +5,7 @@ from testscenarios.testcase import TestWithScenarios
from six.moves import configparser, StringIO
from jenkins_jobs import cmd
from jenkins_jobs.builder import ModuleRegistry
from jenkins_jobs.registry import ModuleRegistry
class ModuleRegistryPluginInfoTestsWithScenarios(TestWithScenarios,