Rewrite YAML parser
Rewrite YAML parser, YAML objects and parameters expansion logic to
enable better control over expansion logic.
Broken backward compatilibity:
* More agressive parameter expansion. This may lead to parameters
expanded in places where they were not expanded before.
* Top-level elements, which is not known to parser (such as 'job',
'view', 'project' etc), are now lead to parse failures.
Prepend them with underscore to be ignored by parser.
* Files included using '!include-raw:' elements and having formatting in
it's path ('lazy-loaded' in previous implementation) are now expanded
too.
Use '!include-raw-escape:' for them instead.
See changes in these tests for examples:
tests/yamlparser/job_fixtures/lazy-load-jobs-multi001.yaml
tests/yamlparser/job_fixtures/lazy-load-jobs-multi002.yaml
tests/yamlparser/job_fixtures/lazy-load-jobs001.yaml
* Parameters with template value using itself were substituted as is.
For example: "timer: '{timer}'" was expanded to "{timer}". Now it
leads to recursive parameter error.
See changes in this test for example:
tests/yamlparser/job_fixtures/parameter_name_reuse_default.*
->
tests/yamlparser/error_fixtures/parameter_name_reuse_default.*
* When job group includes a job which was never declared, it was just
ignored. Now it fails: job is missing.
See changes in this test for example:
tests/yamlparser/job_fixtures/job_group_includes_missing_job.*
->
tests/yamlparser/error_fixtures/job_group_includes_missing_job.*
Change-Id: Ief4e515f065a1b9e0f74fe06d7e94fa77d69f273
changes/65/871965/15
Vsevolod Fedorov
11 months ago
parent
a47e4ee896
commit
af9e03ec08
@ -0,0 +1,187 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
job_contents_keys = {
|
||||
# Same as for macros.
|
||||
"parameters",
|
||||
"properties",
|
||||
"builders",
|
||||
"wrappers",
|
||||
"triggers",
|
||||
"publishers",
|
||||
"scm",
|
||||
"pipeline-scm",
|
||||
"reporters",
|
||||
# General.
|
||||
"project-type",
|
||||
"folder",
|
||||
"node",
|
||||
"jdk",
|
||||
"actions",
|
||||
"disabled",
|
||||
"display-name",
|
||||
"block-downstream",
|
||||
"block-upstream",
|
||||
"auth-token",
|
||||
"concurrent",
|
||||
"workspace",
|
||||
"child-workspace",
|
||||
"quiet-period",
|
||||
"retry-count",
|
||||
"logrotate",
|
||||
"raw",
|
||||
# Builders.
|
||||
"prebuilders",
|
||||
"postbuilders",
|
||||
# HipChat.
|
||||
"hipchat",
|
||||
# Notificatoins.
|
||||
"notifications",
|
||||
# project Flow.
|
||||
"dsl",
|
||||
"needs-workspace",
|
||||
"dsl-file",
|
||||
# GithubOrganization.
|
||||
"prune-dead-branches",
|
||||
"days-to-keep",
|
||||
"number-to-keep",
|
||||
"periodic-folder-trigger",
|
||||
"github-org",
|
||||
"script-path",
|
||||
# Matrix.
|
||||
"execution-strategy",
|
||||
"yaml-strategy",
|
||||
"p4-strategy",
|
||||
"axes",
|
||||
# Maven.
|
||||
"maven",
|
||||
"per-module-email",
|
||||
# WorkflowMultiBranch.
|
||||
"sandbox",
|
||||
"script-id",
|
||||
"script-path",
|
||||
"prune-dead-branches",
|
||||
"days-to-keep",
|
||||
"number-to-keep",
|
||||
"periodic-folder-trigger",
|
||||
# Pipeline.
|
||||
"dsl",
|
||||
"sandbox",
|
||||
# project Workflow.
|
||||
"dsl",
|
||||
"sandbox",
|
||||
}
|
||||
|
||||
view_contents_keys = {
|
||||
# Common.
|
||||
"filter-executors",
|
||||
"filter-queue",
|
||||
# All
|
||||
# <nothing>
|
||||
# List.
|
||||
"job-name",
|
||||
"job-filters",
|
||||
"width",
|
||||
"alignment",
|
||||
"columns",
|
||||
"regex",
|
||||
"recurse",
|
||||
# Sectioned.
|
||||
"sections",
|
||||
# SectionedText.
|
||||
"width",
|
||||
"alignment",
|
||||
"text",
|
||||
"style",
|
||||
# DeliveryPipeline.
|
||||
"aggregated-changes-grouping-pattern",
|
||||
"allow-abort",
|
||||
"allow-manual-triggers",
|
||||
"allow-pipeline-start",
|
||||
"allow-rebuild",
|
||||
"link-relative",
|
||||
"link-to-console-log",
|
||||
"max-number-of-visible-pipelines",
|
||||
"name",
|
||||
"no-of-columns",
|
||||
"no-of-pipelines",
|
||||
"paging-enabled",
|
||||
"show-absolute-date-time",
|
||||
"show-aggregated-changes",
|
||||
"show-aggregated-pipeline",
|
||||
"show-avatars",
|
||||
"show-changes",
|
||||
"show-description",
|
||||
"show-promotions",
|
||||
"show-static-analysis-results",
|
||||
"show-test-results",
|
||||
"show-total-build-time",
|
||||
"update-interval",
|
||||
"sorting",
|
||||
"components",
|
||||
"regexps",
|
||||
# Nested.
|
||||
"views",
|
||||
"default-view",
|
||||
"columns",
|
||||
# Pipeline.
|
||||
"first-job",
|
||||
"name",
|
||||
"no-of-displayed-builds",
|
||||
"title",
|
||||
"link-style",
|
||||
"css-Url",
|
||||
"latest-job-only",
|
||||
"manual-trigger",
|
||||
"show-parameters",
|
||||
"parameters-in-headers",
|
||||
"start-with-parameters",
|
||||
"refresh-frequency",
|
||||
"definition-header",
|
||||
}
|
||||
|
||||
|
||||
def split_contents_params(data, contents_keys):
|
||||
contents = {key: value for key, value in data.items() if key in contents_keys}
|
||||
params = {key: value for key, value in data.items() if key not in contents_keys}
|
||||
return (contents, params)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Defaults:
|
||||
name: str
|
||||
params: dict
|
||||
contents: dict # Values that go to job contents.
|
||||
|
||||
@classmethod
|
||||
def add(cls, config, roots, expander, params_expander, data):
|
||||
d = {**data}
|
||||
name = d.pop("name")
|
||||
contents, params = split_contents_params(
|
||||
d, job_contents_keys | view_contents_keys
|
||||
)
|
||||
defaults = cls(name, params, contents)
|
||||
roots.defaults[name] = defaults
|
||||
|
||||
@classmethod
|
||||
def empty(cls):
|
||||
return Defaults("empty", params={}, contents={})
|
||||
|
||||
def merged_with_global(self, global_):
|
||||
return Defaults(
|
||||
name=f"{self.name}-merged-with-global",
|
||||
params={**global_.params, **self.params},
|
||||
contents={**global_.contents, **self.contents},
|
||||
)
|
||||
@ -0,0 +1,89 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import itertools
|
||||
|
||||
from .errors import JenkinsJobsException
|
||||
|
||||
|
||||
def merge_dicts(dict_list):
|
||||
result = {}
|
||||
for d in dict_list:
|
||||
result.update(d)
|
||||
return result
|
||||
|
||||
|
||||
class DimensionsExpander:
|
||||
def __init__(self, context):
|
||||
self._context = context
|
||||
|
||||
def enum_dimensions_params(self, axes, params, defaults):
|
||||
if not axes:
|
||||
# No axes - instantiate one job/view.
|
||||
yield {}
|
||||
return
|
||||
dim_values = []
|
||||
for axis in axes:
|
||||
try:
|
||||
value = params[axis]
|
||||
except KeyError:
|
||||
try:
|
||||
value = defaults[axis]
|
||||
except KeyError:
|
||||
continue # May be, value would be received from an another axis values.
|
||||
value = self._decode_axis_value(axis, value)
|
||||
dim_values.append(value)
|
||||
for values in itertools.product(*dim_values):
|
||||
yield merge_dicts(values)
|
||||
|
||||
def _decode_axis_value(self, axis, value):
|
||||
if not isinstance(value, list):
|
||||
yield {axis: value}
|
||||
return
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
yield {axis: item}
|
||||
continue
|
||||
if len(item.items()) != 1:
|
||||
raise JenkinsJobsException(
|
||||
f"Invalid parameter {axis!r} definition for template {self._context!r}:"
|
||||
f" Expected a value or a dict with single element, but got: {item!r}"
|
||||
)
|
||||
value, p = next(iter(item.items()))
|
||||
yield {
|
||||
axis: value, # Point axis value.
|
||||
**p, # Point-specific parameters. May override asis value.
|
||||
}
|
||||
|
||||
def is_point_included(self, exclude_list, params):
|
||||
return not any(self._match_exclude(params, el) for el in exclude_list or [])
|
||||
|
||||
def _match_exclude(self, params, exclude):
|
||||
if not isinstance(exclude, dict):
|
||||
raise JenkinsJobsException(
|
||||
f"Template {self._context!r}: Exclude element should be dict, but is: {exclude!r}"
|
||||
)
|
||||
if not exclude:
|
||||
raise JenkinsJobsException(
|
||||
f"Template {self._context!r}: Exclude element should be dict, but is empty: {exclude!r}"
|
||||
)
|
||||
for axis, value in exclude.items():
|
||||
try:
|
||||
v = params[axis]
|
||||
except KeyError:
|
||||
raise JenkinsJobsException(
|
||||
f"Template {self._context!r}: Unknown axis {axis!r} for exclude element: {exclude!r}"
|
||||
)
|
||||
if value != v:
|
||||
return False
|
||||
# All required exclude values are matched.
|
||||
return True
|
||||
@ -0,0 +1,214 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from functools import partial
|
||||
|
||||
from jinja2 import StrictUndefined
|
||||
|
||||
from .errors import JenkinsJobsException
|
||||
from .formatter import CustomFormatter, enum_str_format_required_params
|
||||
from .yaml_objects import (
|
||||
J2String,
|
||||
J2Yaml,
|
||||
YamlInclude,
|
||||
YamlListJoin,
|
||||
IncludeJinja2,
|
||||
IncludeRaw,
|
||||
IncludeRawEscape,
|
||||
)
|
||||
|
||||
|
||||
def expand_dict(expander, obj, params):
|
||||
result = {}
|
||||
for key, value in obj.items():
|
||||
expanded_key = expander.expand(key, params)
|
||||
expanded_value = expander.expand(value, params)
|
||||
result[expanded_key] = expanded_value
|
||||
return result
|
||||
|
||||
|
||||
def expand_list(expander, obj, params):
|
||||
return [expander.expand(item, params) for item in obj]
|
||||
|
||||
|
||||
def expand_tuple(expander, obj, params):
|
||||
return tuple(expander.expand(item, params) for item in obj)
|
||||
|
||||
|
||||
class StrExpander:
|
||||
def __init__(self, config):
|
||||
allow_empty = config.yamlparser["allow_empty_variables"]
|
||||
self._formatter = CustomFormatter(allow_empty)
|
||||
|
||||
def __call__(self, obj, params):
|
||||
return self._formatter.format(obj, **params)
|
||||
|
||||
|
||||
def call_expand(expander, obj, params):
|
||||
return obj.expand(expander, params)
|
||||
|
||||
|
||||
def call_subst(expander, obj, params):
|
||||
return obj.subst(expander, params)
|
||||
|
||||
|
||||
def dont_expand(obj, params):
|
||||
return obj
|
||||
|
||||
|
||||
yaml_classes_list = [
|
||||
J2String,
|
||||
J2Yaml,
|
||||
YamlInclude,
|
||||
YamlListJoin,
|
||||
IncludeJinja2,
|
||||
IncludeRaw,
|
||||
IncludeRawEscape,
|
||||
]
|
||||
|
||||
deprecated_yaml_tags = [
|
||||
("!include", YamlInclude),
|
||||
("!include-raw", IncludeRaw),
|
||||
("!include-raw-escape", IncludeRawEscape),
|
||||
]
|
||||
|
||||
|
||||
# Does not expand string formats. Used in jobs and macros without parameters.
|
||||
class Expander:
|
||||
def __init__(self, config):
|
||||
_yaml_object_expanders = {
|
||||
cls: partial(call_expand, self) for cls in yaml_classes_list
|
||||
}
|
||||
self.expanders = {
|
||||
dict: partial(expand_dict, self),
|
||||
list: partial(expand_list, self),
|
||||
tuple: partial(expand_tuple, self),
|
||||
str: dont_expand,
|
||||
bool: dont_expand,
|
||||
int: dont_expand,
|
||||
float: dont_expand,
|
||||
type(None): dont_expand,
|
||||
**_yaml_object_expanders,
|
||||
}
|
||||
|
||||
def expand(self, obj, params):
|
||||
t = type(obj)
|
||||
try:
|
||||
expander = self.expanders[t]
|
||||
except KeyError:
|
||||
raise RuntimeError(f"Do not know how to expand type: {t!r}")
|
||||
return expander(obj, params)
|
||||
|
||||
|
||||
# Expands string formats also. Used in jobs templates and macros with parameters.
|
||||
class ParamsExpander(Expander):
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
_yaml_object_expanders = {
|
||||
cls: partial(call_subst, self) for cls in yaml_classes_list
|
||||
}
|
||||
self.expanders.update(
|
||||
{
|
||||
str: StrExpander(config),
|
||||
**_yaml_object_expanders,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def call_required_params(obj):
|
||||
yield from obj.required_params
|
||||
|
||||
|
||||
def enum_dict_params(obj):
|
||||
for key, value in obj.items():
|
||||
yield from enum_required_params(key)
|
||||
yield from enum_required_params(value)
|
||||
|
||||
|
||||
def enum_seq_params(obj):
|
||||
for value in obj:
|
||||
yield from enum_required_params(value)
|
||||
|
||||
|
||||
def no_parameters(obj):
|
||||
return []
|
||||
|
||||
|
||||
yaml_classes_enumers = {cls: call_required_params for cls in yaml_classes_list}
|
||||
|
||||
param_enumers = {
|
||||
str: enum_str_format_required_params,
|
||||
dict: enum_dict_params,
|
||||
list: enum_seq_params,
|
||||
tuple: enum_seq_params,
|
||||
bool: no_parameters,
|
||||
int: no_parameters,
|
||||
float: no_parameters,
|
||||
type(None): no_parameters,
|
||||
**yaml_classes_enumers,
|
||||
}
|
||||
|
||||
# Do not expand these.
|
||||
disable_expand_for = {"template-name"}
|
||||
|
||||
|
||||
def enum_required_params(obj):
|
||||
t = type(obj)
|
||||
try:
|
||||
enumer = param_enumers[t]
|
||||
except KeyError:
|
||||
raise RuntimeError(
|
||||
f"Do not know how to enumerate required parameters for type: {t!r}"
|
||||
)
|
||||
return enumer(obj)
|
||||
|
||||
|
||||
def expand_parameters(expander, param_dict, template_name):
|
||||
expanded_params = {}
|
||||
deps = {} # Using dict as ordered set.
|
||||
|
||||
def expand(name):
|
||||
try:
|
||||
return expanded_params[name]
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
format = param_dict[name]
|
||||
except KeyError:
|
||||
return StrictUndefined(name=name)
|
||||
if name in deps:
|
||||
raise RuntimeError(
|
||||
f"While expanding {name!r} for template {template_name!r}:"
|
||||
f" Recursive parameters usage: {name} <- {' <- '.join(deps)}"
|
||||
)
|
||||
if name in disable_expand_for:
|
||||
value = format
|
||||
else:
|
||||
required_params = list(enum_required_params(format))
|
||||
deps[name] = None
|
||||
try:
|
||||
params = {n: expand(n) for n in required_params}
|
||||
finally:
|
||||
deps.popitem()
|
||||
try:
|
||||
value = expander.expand(format, params)
|
||||
except JenkinsJobsException as x:
|
||||
used_by_deps = ", used by".join(f"{d!r}" for d in deps)
|
||||
raise RuntimeError(
|
||||
f"While expanding {name!r}, used by {used_by_deps}, used by template {template_name!r}: {x}"
|
||||
)
|
||||
expanded_params[name] = value
|
||||
return value
|
||||
|
||||
for name in param_dict:
|
||||
expand(name)
|
||||
return expanded_params
|
||||
@ -0,0 +1,107 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group
|
||||
from .defaults import split_contents_params, job_contents_keys
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobBase(RootBase):
|
||||
project_type: str
|
||||
folder: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, config, roots, expander, data):
|
||||
keep_descriptions = config.yamlparser["keep_descriptions"]
|
||||
d = {**data}
|
||||
name = d.pop("name")
|
||||
id = d.pop("id", None)
|
||||
description = d.pop("description", None)
|
||||
defaults = d.pop("defaults", "global")
|
||||
project_type = d.pop("project-type", None)
|
||||
folder = d.pop("folder", None)
|
||||
contents, params = split_contents_params(d, job_contents_keys)
|
||||
return cls(
|
||||
roots.defaults,
|
||||
expander,
|
||||
keep_descriptions,
|
||||
id,
|
||||
name,
|
||||
description,
|
||||
defaults,
|
||||
params,
|
||||
contents,
|
||||
project_type,
|
||||
folder,
|
||||
)
|
||||
|
||||
def _as_dict(self):
|
||||
data = {
|
||||
"name": self._full_name,
|
||||
**self.contents,
|
||||
}
|
||||
if self.project_type:
|
||||
data["project-type"] = self.project_type
|
||||
return data
|
||||
|
||||
@property
|
||||
def _full_name(self):
|
||||
if self.folder:
|
||||
return f"{self.folder}/{self.name}"
|
||||
else:
|
||||
return self.name
|
||||
|
||||
|
||||
class Job(JobBase, NonTemplateRootMixin):
|
||||
@classmethod
|
||||
def add(cls, config, roots, expander, param_expander, data):
|
||||
job = cls.from_dict(config, roots, expander, data)
|
||||
roots.assign(roots.jobs, job.id, job, "job")
|
||||
|
||||
|
||||
class JobTemplate(JobBase, TemplateRootMixin):
|
||||
@classmethod
|
||||
def add(cls, config, roots, expander, params_expander, data):
|
||||
template = cls.from_dict(config, roots, params_expander, data)
|
||||
roots.assign(roots.job_templates, template.id, template, "job template")
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobGroup(Group):
|
||||
_jobs: dict
|
||||
_job_templates: dict
|
||||
|
||||
@classmethod
|
||||
def add(cls, config, roots, expander, params_expander, data):
|
||||
d = {**data}
|
||||
name = d.pop("name")
|
||||
job_specs = [
|
||||
cls._spec_from_dict(item, error_context=f"Job group {name}")
|
||||
for item in d.pop("jobs", [])
|
||||
]
|
||||
group = cls(
|
||||
name,
|
||||
job_specs,
|
||||
d,
|
||||
roots.jobs,
|
||||
roots.job_templates,
|
||||
)
|
||||
roots.assign(roots.job_groups, group.name, group, "job group")
|
||||
|
||||
def __str__(self):
|
||||
return f"Job group {self.name}"
|
||||
|
||||
@property
|
||||
def _root_dicts(self):
|
||||
return [self._jobs, self._job_templates]
|
||||
@ -0,0 +1,151 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import io
|
||||
import logging
|
||||
from functools import partial
|
||||
|
||||
import yaml
|
||||
|
||||
from .errors import JenkinsJobsException
|
||||
from .yaml_objects import BaseYamlObject
|
||||
from .expander import Expander, ParamsExpander, deprecated_yaml_tags, yaml_classes_list
|
||||
from .roots import root_adders
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Loader(yaml.Loader):
|
||||
@classmethod
|
||||
def empty(cls, jjb_config):
|
||||
return cls(io.StringIO(), jjb_config)
|
||||
|
||||
def __init__(self, stream, jjb_config, source_path=None, anchors=None):
|
||||
super().__init__(stream)
|
||||
self.jjb_config = jjb_config
|
||||
self.source_path = source_path
|
||||
self._retain_anchors = jjb_config.yamlparser["retain_anchors"]
|
||||
if anchors:
|
||||
# Override default set by super class.
|
||||
self.anchors = anchors
|
||||
|
||||
# Override the default composer to skip resetting the anchors at the
|
||||
# end of the current document.
|
||||
def compose_document(self):
|
||||
# Drop the DOCUMENT-START event.
|
||||
self.get_event()
|
||||
# Compose the root node.
|
||||
node = self.compose_node(None, None)
|
||||
# Drop the DOCUMENT-END event.
|
||||
self.get_event()
|
||||
return node
|
||||
|
||||
def _with_stream(self, stream, source_path):
|
||||
return Loader(stream, self.jjb_config, source_path, self.anchors)
|
||||
|
||||
def load_fp(self, fp):
|
||||
return self.load(fp)
|
||||
|
||||
def load_path(self, path):
|
||||
return self.load(path.read_text(), source_path=path)
|
||||
|
||||
def load(self, stream, source_path=None):
|
||||
loader = self._with_stream(stream, source_path)
|
||||
try:
|
||||
return loader.get_single_data()
|
||||
finally:
|
||||
loader.dispose()
|
||||
if self._retain_anchors:
|
||||
self.anchors.update(loader.anchors)
|
||||
|
||||
|
||||
def load_deprecated_yaml(tag, cls, loader, node):
|
||||
logger.warning("Tag %r is deprecated, switch to using %r", tag, cls.yaml_tag)
|
||||
return cls.from_yaml(loader, node)
|
||||
|
||||
|
||||
for cls in yaml_classes_list:
|
||||
yaml.add_constructor(cls.yaml_tag, cls.from_yaml, Loader)
|
||||
|
||||
for tag, cls in deprecated_yaml_tags:
|
||||
yaml.add_constructor(tag, partial(load_deprecated_yaml, tag, cls), Loader)
|
||||
|
||||
|
||||
def is_stdin(path):
|
||||
return hasattr(path, "read")
|
||||
|
||||
|
||||
def enum_expanded_paths(path_list):
|
||||
visited_set = set()
|
||||
|
||||
def real(path):
|
||||
real_path = path.resolve()
|
||||
if real_path in visited_set:
|
||||
logger.warning(
|
||||
"File '%s' is already added as '%s'; ignoring reference to avoid"
|
||||
" duplicating YAML definitions.",
|
||||
path,
|
||||
real_path,
|
||||
)
|
||||
else:
|
||||
yield real_path
|
||||
visited_set.add(real_path)
|
||||
|
||||
for path in path_list:
|
||||
if is_stdin(path):
|
||||
yield path
|
||||
elif path.is_dir():
|
||||
for p in path.iterdir():
|
||||
if p.suffix in {".yml", ".yaml"}:
|
||||
yield from real(p)
|
||||
else:
|
||||
yield from real(path)
|
||||
|
||||
|
||||
def load_files(config, roots, path_list):
|
||||
expander = Expander(config)
|
||||
params_expander = ParamsExpander(config)
|
||||
loader = Loader.empty(config)
|
||||
for path in enum_expanded_paths(path_list):
|
||||
if is_stdin(path):
|
||||
data = loader.load_fp(path)
|
||||
else:
|
||||
data = loader.load_path(path)
|
||||
if not isinstance(data, list):
|
||||
raise JenkinsJobsException(
|
||||
f"The topmost collection in file '{path}' must be a list,"
|
||||
f" not a {type(data)}"
|
||||
)
|
||||
for item in data:
|
||||
if not isinstance(item, dict):
|
||||
raise JenkinsJobsException(
|
||||
f"{path}: Topmost list should contain single-item dict,"
|
||||
f" not a {type(item)}. Missing indent?"
|
||||
)
|
||||
if len(item) != 1:
|
||||
raise JenkinsJobsException(
|
||||
f"{path}: Topmost dict should be single-item,"
|
||||
f" but have keys {item.keys()}. Missing indent?"
|
||||
)
|
||||
kind, contents = next(iter(item.items()))
|
||||
if kind.startswith("_"):
|
||||
continue
|
||||
if isinstance(contents, BaseYamlObject):
|
||||
contents = contents.expand(expander, params={})
|
||||
try:
|
||||
adder = root_adders[kind]
|
||||
except KeyError:
|
||||
raise JenkinsJobsException(
|
||||
f"{path}: Unknown topmost element type : {kind!r},"
|
||||
f" Known are: {','.join(root_adders)}."
|
||||
)
|
||||
adder(config, roots, expander, params_expander, contents)
|
||||
@ -1,676 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (C) 2013 Hewlett-Packard.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Provides local yaml parsing classes and extend yaml module
|
||||
|
||||
"""Custom application specific yamls tags are supported to provide
|
||||
enhancements when reading yaml configuration.
|
||||
|
||||
Action Tags
|
||||
^^^^^^^^^^^
|
||||
|
||||
These allow manipulation of data being stored in one layout in the source
|
||||
yaml for convenience and/or clarity, to another format to be processed by
|
||||
the targeted module instead of requiring all modules in JJB being capable
|
||||
of supporting multiple input formats.
|
||||
|
||||
The tag ``!join:`` will treat the first element of the following list as
|
||||
the delimiter to use, when joining the remaining elements into a string
|
||||
and returning a single string to be consumed by the specified module option.
|
||||
|
||||
This allows users to maintain elements of data in a list structure for ease
|
||||
of review/maintenance, and have the yaml parser convert it to a string for
|
||||
consumption as any argument for modules. The main expected use case is to
|
||||
allow for generic plugin data such as shell properties to be populated from
|
||||
a list construct which the yaml parser converts to a single string, instead
|
||||
of trying to support this within the module code which would require a
|
||||
templating engine similar to Jinja.
|
||||
|
||||
Generic Example:
|
||||
|
||||
.. literalinclude:: /../../tests/localyaml/fixtures/joinlists.yaml
|
||||
|
||||
|
||||
Environment Inject:
|
||||
|
||||
.. literalinclude:: /../../tests/yamlparser/job_fixtures/string_join.yaml
|
||||
|
||||
|
||||
While this mechanism can also be used items where delimiters are supported by
|
||||
the module, that should be considered a bug that the existing code doesn't
|
||||
handle being provided a list and delimiter to perform the correct conversion
|
||||
for you. Should you discover a module that takes arguments with delimiters and
|
||||
the existing JJB codebase does not handle accepting lists, then this can be
|
||||
used as a temporary solution in place of using very long strings:
|
||||
|
||||
Extended Params Example:
|
||||
|
||||
.. literalinclude::
|
||||
/../../tests/parameters/fixtures/extended-choice-param-full.yaml
|
||||
|
||||
|
||||
Inclusion Tags
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
These allow inclusion of arbitrary files as a method of having blocks of data
|
||||
managed separately to the yaml job configurations. A specific usage of this is
|
||||
inlining scripts contained in separate files, although such tags may also be
|
||||
used to simplify usage of macros or job templates.
|
||||
|
||||
The tag ``!include:`` will treat the following string as file which should be
|
||||
parsed as yaml configuration data.
|
||||
|
||||
Example:
|
||||
|
||||
.. literalinclude:: /../../tests/localyaml/fixtures/include001.yaml
|
||||
|
||||
contents of include001.yaml.inc:
|
||||
|
||||
.. literalinclude:: /../../tests/yamlparser/job_fixtures/include001.yaml.inc
|
||||
|
||||
|
||||
The tag ``!include-raw:`` will treat the given string or list of strings as
|
||||
filenames to be opened as one or more data blob, which should be read into
|
||||
the calling yaml construct without any further parsing. Any data in a file
|
||||
included through this tag, will be treated as string data.
|
||||
|
||||
Examples:
|
||||
|
||||
.. literalinclude:: /../../tests/localyaml/fixtures/include-raw001.yaml
|
||||
|
||||
contents of include-raw001-hello-world.sh:
|
||||
|
||||
.. literalinclude::
|
||||
/../../tests/localyaml/fixtures/include-raw001-hello-world.sh
|
||||
|
||||
contents of include-raw001-vars.sh:
|
||||
|
||||
.. literalinclude::
|
||||
/../../tests/localyaml/fixtures/include-raw001-vars.sh
|
||||
|
||||
using a list of files:
|
||||
|
||||
.. literalinclude::
|
||||
/../../tests/localyaml/fixtures/include-raw-multi001.yaml
|
||||
|
||||
The tag ``!include-raw-escape:`` treats the given string or list of strings as
|
||||
filenames to be opened as one or more data blobs, which should be escaped
|
||||
before being read in as string data. This allows job-templates to use this tag
|
||||
to include scripts from files without needing to escape braces in the original
|
||||
file.
|
||||
|
||||
.. warning::
|
||||
|
||||
When used as a macro ``!include-raw-escape:`` should only be used if
|
||||
parameters are passed into the escaped file and you would like to escape
|
||||
those parameters. If the file does not have any jjb parameters passed into
|
||||
it then ``!include-raw:`` should be used instead otherwise you will run
|
||||
into an interesting issue where ``include-raw-escape:`` actually adds
|
||||
additional curly braces around existing curly braces. For example
|
||||
${PROJECT} becomes ${{PROJECT}} which may break bash scripts.
|
||||
|
||||
Examples:
|
||||
|
||||
.. literalinclude::
|
||||
/../../tests/localyaml/fixtures/include-raw-escaped001.yaml
|
||||
|
||||
contents of include-raw001-hello-world.sh:
|
||||
|
||||
.. literalinclude::
|
||||
/../../tests/localyaml/fixtures/include-raw001-hello-world.sh
|
||||
|
||||
contents of include-raw001-vars.sh:
|
||||
|
||||
.. literalinclude::
|
||||
/../../tests/localyaml/fixtures/include-raw001-vars.sh
|
||||
|
||||
using a list of files:
|
||||
|
||||
.. literalinclude::
|
||||
/../../tests/localyaml/fixtures/include-raw-escaped-multi001.yaml
|
||||