591 lines
22 KiB
Python
591 lines
22 KiB
Python
#!/usr/bin/env python
|
|
# Copyright (C) 2015 OpenStack, LLC.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# Manage JJB yaml feature implementation
|
|
|
|
import copy
|
|
import fnmatch
|
|
import io
|
|
import itertools
|
|
import logging
|
|
import re
|
|
import os
|
|
|
|
from jenkins_jobs.constants import MAGIC_MANAGE_STRING
|
|
from jenkins_jobs.errors import JenkinsJobsException
|
|
from jenkins_jobs.formatter import deep_format
|
|
import jenkins_jobs.local_yaml as local_yaml
|
|
from jenkins_jobs import utils
|
|
|
|
__all__ = ["YamlParser"]
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def matches(what, glob_patterns):
|
|
"""
|
|
Checks if the given string, ``what``, matches any of the glob patterns in
|
|
the iterable, ``glob_patterns``
|
|
|
|
:arg str what: String that we want to test if it matches a pattern
|
|
:arg iterable glob_patterns: glob patterns to match (list, tuple, set,
|
|
etc.)
|
|
"""
|
|
return any(fnmatch.fnmatch(what, glob_pattern) for glob_pattern in glob_patterns)
|
|
|
|
|
|
def combination_matches(combination, match_combinations):
|
|
"""
|
|
Checks if the given combination is matches for any of the given combination
|
|
globs, being those a set of combinations where if a key is missing, it's
|
|
considered matching
|
|
|
|
(key1=2, key2=3)
|
|
|
|
would match the combination match:
|
|
(key2=3)
|
|
|
|
but not:
|
|
(key1=2, key2=2)
|
|
"""
|
|
for cmatch in match_combinations:
|
|
for key, val in combination.items():
|
|
if cmatch.get(key, val) != val:
|
|
break
|
|
else:
|
|
return True
|
|
return False
|
|
|
|
|
|
class YamlParser(object):
|
|
def __init__(self, jjb_config=None):
|
|
self.data = {}
|
|
self.jobs = []
|
|
self.views = []
|
|
|
|
self.jjb_config = jjb_config
|
|
self.keep_desc = jjb_config.yamlparser["keep_descriptions"]
|
|
self.path = jjb_config.yamlparser["include_path"]
|
|
|
|
def load_files(self, fn):
|
|
|
|
# handle deprecated behavior, and check that it's not a file like
|
|
# object as these may implement the '__iter__' attribute.
|
|
if not hasattr(fn, "__iter__") or hasattr(fn, "read"):
|
|
logger.warning(
|
|
"Passing single elements for the `fn` argument in "
|
|
"Builder.load_files is deprecated. Please update your code "
|
|
"to use a list as support for automatic conversion will be "
|
|
"removed in a future version."
|
|
)
|
|
fn = [fn]
|
|
|
|
files_to_process = []
|
|
for path in fn:
|
|
if not hasattr(path, "read") and os.path.isdir(path):
|
|
files_to_process.extend(
|
|
[
|
|
os.path.join(path, f)
|
|
for f in sorted(os.listdir(path))
|
|
if (f.endswith(".yml") or f.endswith(".yaml"))
|
|
]
|
|
)
|
|
else:
|
|
files_to_process.append(path)
|
|
|
|
# symlinks used to allow loading of sub-dirs can result in duplicate
|
|
# definitions of macros and templates when loading all from top-level
|
|
unique_files = []
|
|
for f in files_to_process:
|
|
if hasattr(f, "read"):
|
|
unique_files.append(f)
|
|
continue
|
|
rpf = os.path.realpath(f)
|
|
if rpf not in unique_files:
|
|
unique_files.append(rpf)
|
|
else:
|
|
logger.warning(
|
|
"File '%s' already added as '%s', ignoring "
|
|
"reference to avoid duplicating yaml "
|
|
"definitions." % (f, rpf)
|
|
)
|
|
|
|
for in_file in unique_files:
|
|
# use of ask-for-permissions instead of ask-for-forgiveness
|
|
# performs better when low use cases.
|
|
if hasattr(in_file, "name"):
|
|
fname = in_file.name
|
|
else:
|
|
fname = in_file
|
|
logger.debug("Parsing YAML file {0}".format(fname))
|
|
if hasattr(in_file, "read"):
|
|
self._parse_fp(in_file)
|
|
else:
|
|
self.parse(in_file)
|
|
|
|
def _parse_fp(self, fp):
|
|
# wrap provided file streams to ensure correct encoding used
|
|
data = local_yaml.load(
|
|
utils.wrap_stream(fp),
|
|
self.jjb_config.yamlparser["retain_anchors"],
|
|
search_path=self.path,
|
|
)
|
|
if data:
|
|
if not isinstance(data, list):
|
|
raise JenkinsJobsException(
|
|
"The topmost collection in file '{fname}' must be a list,"
|
|
" not a {cls}".format(fname=getattr(fp, "name", fp), cls=type(data))
|
|
)
|
|
for item in data:
|
|
cls, dfn = next(iter(item.items()))
|
|
group = self.data.get(cls, {})
|
|
if len(item.items()) > 1:
|
|
n = None
|
|
for k, v in item.items():
|
|
if k == "name":
|
|
n = v
|
|
break
|
|
# Syntax error
|
|
raise JenkinsJobsException(
|
|
"Syntax error, for item "
|
|
"named '{0}'. Missing indent?".format(n)
|
|
)
|
|
# allow any entry to specify an id that can also be used
|
|
_id = dfn.get("id", dfn["name"])
|
|
if _id in group:
|
|
self._handle_dups(
|
|
"Duplicate entry found in '{0}: '{1}' already "
|
|
"defined".format(fp.name, _id)
|
|
)
|
|
group[_id] = dfn
|
|
self.data[cls] = group
|
|
|
|
def parse(self, fn):
|
|
with io.open(fn, "r", encoding="utf-8") as fp:
|
|
self._parse_fp(fp)
|
|
|
|
def _handle_dups(self, message):
|
|
|
|
if not self.jjb_config.yamlparser["allow_duplicates"]:
|
|
logger.error(message)
|
|
raise JenkinsJobsException(message)
|
|
else:
|
|
logger.warning(message)
|
|
|
|
def _getJob(self, name):
|
|
job = self.data.get("job", {}).get(name, None)
|
|
if not job:
|
|
return job
|
|
return self._applyDefaults(job)
|
|
|
|
def _getJobGroup(self, name):
|
|
return self.data.get("job-group", {}).get(name, None)
|
|
|
|
def _getJobTemplate(self, name):
|
|
job = self.data.get("job-template", {}).get(name, None)
|
|
if not job:
|
|
return job
|
|
return self._applyDefaults(job)
|
|
|
|
def _applyDefaults(self, data, override_dict=None):
|
|
if override_dict is None:
|
|
override_dict = {}
|
|
|
|
whichdefaults = data.get("defaults", "global")
|
|
defaults = copy.deepcopy(self.data.get("defaults", {}).get(whichdefaults, {}))
|
|
if defaults == {} and whichdefaults != "global":
|
|
raise JenkinsJobsException(
|
|
"Unknown defaults set: '{0}'".format(whichdefaults)
|
|
)
|
|
|
|
for key in override_dict.keys():
|
|
if key in defaults.keys():
|
|
defaults[key] = override_dict[key]
|
|
|
|
newdata = {}
|
|
newdata.update(defaults)
|
|
newdata.update(data)
|
|
return newdata
|
|
|
|
def _formatDescription(self, job):
|
|
if self.keep_desc:
|
|
description = job.get("description", None)
|
|
else:
|
|
description = job.get("description", "")
|
|
if description is not None:
|
|
job["description"] = description + self._get_managed_string().lstrip()
|
|
|
|
def _getfullname(self, data):
|
|
if "folder" in data:
|
|
return "%s/%s" % (data["folder"], data["name"])
|
|
|
|
return data["name"]
|
|
|
|
def expandYaml(self, registry, jobs_glob=None):
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for module in registry.modules:
|
|
if hasattr(module, "handle_data"):
|
|
if module.handle_data(self.data):
|
|
changed = True
|
|
|
|
for job in self.data.get("job", {}).values():
|
|
job = self._applyDefaults(job)
|
|
job["name"] = self._getfullname(job)
|
|
|
|
if jobs_glob and not matches(job["name"], jobs_glob):
|
|
logger.debug("Ignoring job {0}".format(job["name"]))
|
|
continue
|
|
logger.debug("Expanding job '{0}'".format(job["name"]))
|
|
self._formatDescription(job)
|
|
self.jobs.append(job)
|
|
|
|
for view in self.data.get("view", {}).values():
|
|
view["name"] = self._getfullname(view)
|
|
|
|
if jobs_glob and not matches(view["name"], jobs_glob):
|
|
logger.debug("Ignoring view {0}".format(view["name"]))
|
|
continue
|
|
|
|
logger.debug("Expanding view '{0}'".format(view["name"]))
|
|
self._formatDescription(view)
|
|
self.views.append(view)
|
|
|
|
for project in self.data.get("project", {}).values():
|
|
logger.debug("Expanding project '{0}'".format(project["name"]))
|
|
# use a set to check for duplicate job references in projects
|
|
seen = set()
|
|
for jobspec in project.get("jobs", []):
|
|
if isinstance(jobspec, dict):
|
|
# Singleton dict containing dict of job-specific params
|
|
jobname, jobparams = next(iter(jobspec.items()))
|
|
if not isinstance(jobparams, dict):
|
|
jobparams = {}
|
|
else:
|
|
jobname = jobspec
|
|
jobparams = {}
|
|
job = self._getJob(jobname)
|
|
if job:
|
|
# Just naming an existing defined job
|
|
if jobname in seen:
|
|
self._handle_dups(
|
|
"Duplicate job '{0}' specified "
|
|
"for project '{1}'".format(jobname, project["name"])
|
|
)
|
|
seen.add(jobname)
|
|
continue
|
|
# see if it's a job group
|
|
group = self._getJobGroup(jobname)
|
|
if group:
|
|
for group_jobspec in group["jobs"]:
|
|
if isinstance(group_jobspec, dict):
|
|
group_jobname, group_jobparams = next(
|
|
iter(group_jobspec.items())
|
|
)
|
|
if not isinstance(group_jobparams, dict):
|
|
group_jobparams = {}
|
|
else:
|
|
group_jobname = group_jobspec
|
|
group_jobparams = {}
|
|
job = self._getJob(group_jobname)
|
|
if job:
|
|
if group_jobname in seen:
|
|
self._handle_dups(
|
|
"Duplicate job '{0}' specified for "
|
|
"project '{1}'".format(
|
|
group_jobname, project["name"]
|
|
)
|
|
)
|
|
seen.add(group_jobname)
|
|
continue
|
|
template = self._getJobTemplate(group_jobname)
|
|
# Allow a group to override parameters set by a project
|
|
d = type(project)(project)
|
|
d.update(jobparams)
|
|
d.update(group)
|
|
d.update(group_jobparams)
|
|
# Except name, since the group's name is not useful
|
|
d["name"] = project["name"]
|
|
if template:
|
|
self._expandYamlForTemplateJob(d, template, jobs_glob)
|
|
continue
|
|
# see if it's a template
|
|
template = self._getJobTemplate(jobname)
|
|
if template:
|
|
d = type(project)(project)
|
|
d.update(jobparams)
|
|
self._expandYamlForTemplateJob(d, template, jobs_glob)
|
|
else:
|
|
raise JenkinsJobsException(
|
|
"Failed to find suitable "
|
|
"template named '{0}'".format(jobname)
|
|
)
|
|
|
|
for viewspec in project.get("views", []):
|
|
if isinstance(viewspec, dict):
|
|
# Singleton dict containing dict of view-specific params
|
|
viewname, viewparams = next(iter(viewspec.items()))
|
|
if not isinstance(viewparams, dict):
|
|
viewparams = {}
|
|
else:
|
|
viewname = viewspec
|
|
viewparams = {}
|
|
view = self._getView(viewname)
|
|
if view:
|
|
# Just naming an existing defined view
|
|
if viewname in seen:
|
|
self._handle_dups(
|
|
"Duplicate view '{0}' specified "
|
|
"for project '{1}'".format(viewname, project["name"])
|
|
)
|
|
seen.add(viewname)
|
|
continue
|
|
# see if it's a view group
|
|
group = self._getViewGroup(viewname)
|
|
if group:
|
|
for group_viewspec in group["views"]:
|
|
if isinstance(group_viewspec, dict):
|
|
group_viewname, group_viewparams = next(
|
|
iter(group_viewspec.items())
|
|
)
|
|
if not isinstance(group_viewparams, dict):
|
|
group_viewparams = {}
|
|
else:
|
|
group_viewname = group_viewspec
|
|
group_viewparams = {}
|
|
view = self._getView(group_viewname)
|
|
if view:
|
|
if group_viewname in seen:
|
|
self._handle_dups(
|
|
"Duplicate view '{0}' specified for "
|
|
"project '{1}'".format(
|
|
group_viewname, project["name"]
|
|
)
|
|
)
|
|
seen.add(group_viewname)
|
|
continue
|
|
template = self._getViewTemplate(group_viewname)
|
|
# Allow a group to override parameters set by a project
|
|
d = type(project)(project)
|
|
d.update(viewparams)
|
|
d.update(group)
|
|
d.update(group_viewparams)
|
|
# Except name, since the group's name is not useful
|
|
d["name"] = project["name"]
|
|
if template:
|
|
self._expandYamlForTemplateView(d, template, jobs_glob)
|
|
continue
|
|
# see if it's a template
|
|
template = self._getViewTemplate(viewname)
|
|
if template:
|
|
d = type(project)(project)
|
|
d.update(viewparams)
|
|
self._expandYamlForTemplateView(d, template, jobs_glob)
|
|
else:
|
|
raise JenkinsJobsException(
|
|
"Failed to find suitable "
|
|
"template named '{0}'".format(viewname)
|
|
)
|
|
|
|
# check for duplicate generated jobs
|
|
seen = set()
|
|
# walk the list in reverse so that last definition wins
|
|
for job in self.jobs[::-1]:
|
|
if job["name"] in seen:
|
|
self._handle_dups(
|
|
"Duplicate definitions for job '{0}' "
|
|
"specified".format(job["name"])
|
|
)
|
|
self.jobs.remove(job)
|
|
seen.add(job["name"])
|
|
|
|
# check for duplicate generated views
|
|
seen_views = set()
|
|
# walk the list in reverse so that last definition wins
|
|
for view in self.views[::-1]:
|
|
if view["name"] in seen_views:
|
|
self._handle_dups(
|
|
"Duplicate definitions for view '{0}' "
|
|
"specified".format(view["name"])
|
|
)
|
|
self.views.remove(view)
|
|
seen_views.add(view["name"])
|
|
|
|
return self.jobs, self.views
|
|
|
|
def _expandYamlForTemplateJob(self, project, template, jobs_glob=None):
|
|
dimensions = []
|
|
template_name = template["name"]
|
|
# reject keys that are not useful during yaml expansion
|
|
for k in ["jobs"]:
|
|
project.pop(k)
|
|
excludes = project.pop("exclude", [])
|
|
for (k, v) in project.items():
|
|
tmpk = "{{{0}}}".format(k)
|
|
if tmpk not in template_name:
|
|
continue
|
|
if type(v) == list:
|
|
dimensions.append(zip([k] * len(v), v))
|
|
# XXX somewhat hackish to ensure we actually have a single
|
|
# pass through the loop
|
|
if len(dimensions) == 0:
|
|
dimensions = [(("", ""),)]
|
|
|
|
for values in itertools.product(*dimensions):
|
|
params = copy.deepcopy(project)
|
|
params = self._applyDefaults(params, template)
|
|
params["template-name"] = re.sub(r"({|})", r"\1\1", template_name)
|
|
|
|
try:
|
|
expanded_values = {}
|
|
for (k, v) in values:
|
|
if isinstance(v, dict):
|
|
inner_key = next(iter(v))
|
|
expanded_values[k] = inner_key
|
|
expanded_values.update(v[inner_key])
|
|
else:
|
|
expanded_values[k] = v
|
|
except TypeError:
|
|
project_name = project.pop("name")
|
|
logger.error(
|
|
"Exception thrown while expanding template '%s' for "
|
|
"project '%s', with expansion arguments of:\n%s\n"
|
|
"Original project input variables for template:\n%s\n"
|
|
"Most likely the inputs have items indented incorrectly "
|
|
"to describe how they should be applied.\n\nNote yaml "
|
|
"'null' is mapped to python's 'None'",
|
|
template_name,
|
|
project_name,
|
|
"".join(
|
|
local_yaml.dump({k: v}, default_flow_style=False)
|
|
for (k, v) in values
|
|
),
|
|
local_yaml.dump(project, default_flow_style=False),
|
|
)
|
|
raise
|
|
|
|
params.update(expanded_values)
|
|
try:
|
|
params = deep_format(params, params)
|
|
except Exception:
|
|
logging.error("Failure formatting params '%s' with itself", params)
|
|
raise
|
|
if combination_matches(params, excludes):
|
|
logger.debug("Excluding combination %s", str(params))
|
|
continue
|
|
|
|
for key in template.keys():
|
|
if key not in params:
|
|
params[key] = template[key]
|
|
|
|
try:
|
|
expanded = deep_format(
|
|
template,
|
|
params,
|
|
self.jjb_config.yamlparser["allow_empty_variables"],
|
|
)
|
|
except Exception:
|
|
logging.error(
|
|
"Failure formatting template '%s', containing '%s' with "
|
|
"params '%s'",
|
|
template_name,
|
|
template,
|
|
params,
|
|
)
|
|
raise
|
|
expanded["name"] = self._getfullname(expanded)
|
|
|
|
job_name = expanded.get("name")
|
|
if jobs_glob and not matches(job_name, jobs_glob):
|
|
continue
|
|
|
|
self._formatDescription(expanded)
|
|
self.jobs.append(expanded)
|
|
|
|
def _get_managed_string(self):
|
|
# The \n\n is not hard coded, because they get stripped if the
|
|
# project does not otherwise have a description.
|
|
return "\n\n" + MAGIC_MANAGE_STRING
|
|
|
|
# Views related
|
|
def _getView(self, name):
|
|
view = self.data.get("view", {}).get(name, None)
|
|
if not view:
|
|
return view
|
|
return self._applyDefaults(view)
|
|
|
|
def _getViewGroup(self, name):
|
|
return self.data.get("view-group", {}).get(name, None)
|
|
|
|
def _getViewTemplate(self, name):
|
|
view = self.data.get("view-template", {}).get(name, None)
|
|
if not view:
|
|
return view
|
|
return self._applyDefaults(view)
|
|
|
|
def _expandYamlForTemplateView(self, project, template, views_glob=None):
|
|
dimensions = []
|
|
template_name = template["name"]
|
|
# reject keys that are not useful during yaml expansion
|
|
for k in ["views"]:
|
|
project.pop(k)
|
|
excludes = project.pop("exclude", [])
|
|
for (k, v) in project.items():
|
|
tmpk = "{{{0}}}".format(k)
|
|
if tmpk not in template_name:
|
|
continue
|
|
if type(v) == list:
|
|
dimensions.append(zip([k] * len(v), v))
|
|
# XXX somewhat hackish to ensure we actually have a single
|
|
# pass through the loop
|
|
if len(dimensions) == 0:
|
|
dimensions = [(("", ""),)]
|
|
|
|
for values in itertools.product(*dimensions):
|
|
params = copy.deepcopy(project)
|
|
params = self._applyDefaults(params, template)
|
|
|
|
expanded_values = {}
|
|
for (k, v) in values:
|
|
if isinstance(v, dict):
|
|
inner_key = next(iter(v))
|
|
expanded_values[k] = inner_key
|
|
expanded_values.update(v[inner_key])
|
|
else:
|
|
expanded_values[k] = v
|
|
|
|
params.update(expanded_values)
|
|
params = deep_format(params, params)
|
|
if combination_matches(params, excludes):
|
|
logger.debug("Excluding combination %s", str(params))
|
|
continue
|
|
|
|
for key in template.keys():
|
|
if key not in params:
|
|
params[key] = template[key]
|
|
|
|
params["template-name"] = template_name
|
|
expanded = deep_format(
|
|
template, params, self.jjb_config.yamlparser["allow_empty_variables"]
|
|
)
|
|
|
|
view_name = expanded.get("name")
|
|
if views_glob and not matches(view_name, views_glob):
|
|
continue
|
|
|
|
self._formatDescription(expanded)
|
|
self.views.append(expanded)
|