Unify variable and tags expansion inside job and macros

Expand variables inside macros without parameters and jobs
the same way as they are expanded inside macros with parameters
and job templates.
Make tags behave inside macros without parameters and jobs
the same way as they are expanded inside macros with parameters
and job templates.
Update or fix affected tests.

Story: 2010588
Story: 2010963
Story: 2010535
Task: 47394
Task: 49069
Task: 47151

Change-Id: Ie05ae6aa386c62ebbf68dd3e2c7001a4e444a47a
This commit is contained in:
Vsevolod Fedorov 2023-11-13 12:28:57 +03:00
parent c162bffbd2
commit 918f1bb925
30 changed files with 109 additions and 132 deletions

View File

@ -174,7 +174,7 @@ class Defaults:
contents: dict # Values that go to job contents.
@classmethod
def add(cls, config, roots, expander, params_expander, data, pos):
def add(cls, config, roots, data, pos):
d = data.copy()
name = d.pop_required_loc_string("name")
contents, params = split_contents_params(

View File

@ -15,7 +15,7 @@ from collections import namedtuple
from .errors import Context, JenkinsJobsException
from .loc_loader import LocList, LocDict
from jenkins_jobs.expander import Expander
from jenkins_jobs.expander import YamlObjectsExpander
Dimension = namedtuple("Dimension", "axis params")
@ -48,7 +48,7 @@ def _decode_axis_value(axis, value, key_pos, value_pos):
def enum_dimensions_params(axes, params, defaults):
expander = Expander()
expander = YamlObjectsExpander()
if not axes:
# No axes - instantiate one job/view.
yield {}
@ -60,9 +60,10 @@ def enum_dimensions_params(axes, params, defaults):
except KeyError:
try:
value = defaults[axis]
key_pos = value_pos = None
except KeyError:
continue # May be, value would be received from an another axis values.
expanded_value = expander.expand(value, params)
expanded_value = expander.expand(value, params, key_pos, value_pos)
value = [
Dimension(axis, params)
for params in _decode_axis_value(axis, expanded_value, key_pos, value_pos)

View File

@ -55,9 +55,8 @@ def expand_tuple(expander, obj, params, key_pos, value_pos):
class StrExpander:
def __init__(self, config):
allow_empty = config.yamlparser["allow_empty_variables"]
self._formatter = CustomFormatter(allow_empty)
def __init__(self, allow_empty_variables):
self._formatter = CustomFormatter(allow_empty_variables)
def __call__(self, obj, params, key_pos, value_pos):
try:
@ -79,14 +78,14 @@ def call_expand(expander, obj, params, key_pos, value_pos):
return obj.expand(expander, params)
def call_subst(expander, obj, params, key_pos, value_pos):
return obj.subst(expander, params)
def dont_expand(obj, params, key_pos, value_pos):
return obj
def dont_expand_yaml_object(expander, obj, params, key_pos, value_pos):
return obj
yaml_classes_list = [
J2String,
J2Yaml,
@ -104,9 +103,13 @@ deprecated_yaml_tags = [
]
# Does not expand string formats. Used in jobs and macros without parameters.
# Expand strings and yaml objects.
class Expander:
def __init__(self, config=None):
if config:
allow_empty_variables = config.yamlparser["allow_empty_variables"]
else:
allow_empty_variables = False
_yaml_object_expanders = {
cls: partial(call_expand, self) for cls in yaml_classes_list
}
@ -116,8 +119,8 @@ class Expander:
list: partial(expand_list, self),
LocList: partial(expand_list, self),
tuple: partial(expand_tuple, self),
str: dont_expand,
LocString: dont_expand,
str: StrExpander(allow_empty_variables),
LocString: StrExpander(allow_empty_variables),
bool: dont_expand,
int: dont_expand,
float: dont_expand,
@ -136,20 +139,26 @@ class Expander:
return expander(obj, params, key_pos, value_pos)
# Expands string formats also. Used in jobs templates and macros with parameters.
class ParamsExpander(Expander):
# Expand only yaml objects.
class YamlObjectsExpander(Expander):
def __init__(self):
super().__init__()
self.expanders.update(
{
str: dont_expand,
LocString: dont_expand,
}
)
# Expand only string parameters.
class StringsOnlyExpander(Expander):
def __init__(self, config):
super().__init__(config)
_yaml_object_expanders = {
cls: partial(call_subst, self) for cls in yaml_classes_list
cls: partial(dont_expand_yaml_object, self) for cls in yaml_classes_list
}
self.expanders.update(
{
str: StrExpander(config),
LocString: StrExpander(config),
**_yaml_object_expanders,
}
)
self.expanders.update(_yaml_object_expanders)
def call_required_params(obj, pos):

View File

@ -14,6 +14,7 @@ from dataclasses import dataclass
from .errors import JenkinsJobsException
from .loc_loader import LocDict
from .expander import Expander
from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group
from .defaults import split_contents_params, job_contents_keys
@ -24,7 +25,7 @@ class JobBase(RootBase):
folder: str
@classmethod
def from_dict(cls, config, roots, expander, data, pos):
def from_dict(cls, config, roots, data, pos):
keep_descriptions = config.yamlparser["keep_descriptions"]
d = data.copy()
name = d.pop_required_loc_string("name")
@ -36,7 +37,7 @@ class JobBase(RootBase):
contents, params = split_contents_params(d, job_contents_keys)
return cls(
roots.defaults,
expander,
Expander(config),
keep_descriptions,
id,
name,
@ -68,8 +69,8 @@ class JobBase(RootBase):
class Job(JobBase, NonTemplateRootMixin):
@classmethod
def add(cls, config, roots, expander, param_expander, data, pos):
job = cls.from_dict(config, roots, expander, data, pos)
def add(cls, config, roots, data, pos):
job = cls.from_dict(config, roots, data, pos)
roots.assign(roots.jobs, job.id, job, "job")
def __str__(self):
@ -78,8 +79,8 @@ class Job(JobBase, NonTemplateRootMixin):
class JobTemplate(JobBase, TemplateRootMixin):
@classmethod
def add(cls, config, roots, expander, params_expander, data, pos):
template = cls.from_dict(config, roots, params_expander, data, pos)
def add(cls, config, roots, data, pos):
template = cls.from_dict(config, roots, data, pos)
roots.assign(roots.job_templates, template.id, template, "job template")
def __str__(self):
@ -92,7 +93,7 @@ class JobGroup(Group):
_job_templates: dict
@classmethod
def add(cls, config, roots, expander, params_expander, data, pos):
def add(cls, config, roots, data, pos):
d = data.copy()
name = d.pop_required_loc_string("name")
try:

View File

@ -17,7 +17,7 @@ from functools import partial
from .errors import JenkinsJobsException
from .loc_loader import LocLoader
from .yaml_objects import BaseYamlObject
from .expander import Expander, ParamsExpander, deprecated_yaml_tags, yaml_classes_list
from .expander import Expander, deprecated_yaml_tags, yaml_classes_list
from .roots import root_adders
logger = logging.getLogger(__name__)
@ -112,7 +112,6 @@ def enum_expanded_paths(path_list):
def load_files(config, roots, path_list):
expander = Expander(config)
params_expander = ParamsExpander(config)
loader = Loader.empty(config)
for path in enum_expanded_paths(path_list):
if is_stdin(path):
@ -152,4 +151,4 @@ def load_files(config, roots, path_list):
f" known are: {','.join(root_adders)}.",
pos=item.pos,
)
adder(config, roots, expander, params_expander, contents, item.pos)
adder(config, roots, contents, item.pos)

View File

@ -44,8 +44,6 @@ class Macro:
elements_name,
config,
roots,
expander,
params_expander,
data,
pos,
):

View File

@ -33,7 +33,7 @@ class Project(GroupBase):
params: dict
@classmethod
def add(cls, config, roots, expander, params_expander, data, pos):
def add(cls, config, roots, data, pos):
d = data.copy()
name = d.pop_required_loc_string("name")
defaults = d.pop_loc_string("defaults", None)

View File

@ -25,7 +25,7 @@ import types
from six import PY2
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.expander import Expander, ParamsExpander
from jenkins_jobs.expander import Expander, StringsOnlyExpander
from jenkins_jobs.yaml_objects import BaseYamlObject
__all__ = ["ModuleRegistry"]
@ -47,7 +47,7 @@ class ModuleRegistry(object):
self.masked_warned = {}
self._macros = {}
self._expander = Expander(jjb_config)
self._params_expander = ParamsExpander(jjb_config)
self._str_expander = StringsOnlyExpander(jjb_config)
if plugins_list is None:
self.plugins_dict = {}
@ -272,20 +272,16 @@ class ModuleRegistry(object):
"component type that is masking an inbuilt "
"definition" % (name, component_type)
)
# Expand macro strings only if at least one macro parameter is provided.
if component_data:
expander = self._params_expander
else:
expander = self._expander
component_data = {} # It may be None.
if component_data is None:
component_data = {}
expander_params = {**component_data, **(job_data or {})}
elements = macro.elements
if isinstance(elements, BaseYamlObject):
# Expand !j2-yaml element right below macro body.
elements = elements.expand(expander, expander_params)
elements = elements.expand(self._str_expander, expander_params)
for b in elements:
try:
element = expander.expand(b, expander_params)
element = self._expander.expand(b, expander_params)
except JenkinsJobsException as x:
raise x.with_context(
f"While expanding macro {name!r}",

View File

@ -14,6 +14,7 @@ from dataclasses import dataclass
from .errors import JenkinsJobsException
from .loc_loader import LocDict
from .expander import Expander
from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group
from .defaults import split_contents_params, view_contents_keys
@ -23,7 +24,7 @@ class ViewBase(RootBase):
view_type: str
@classmethod
def from_dict(cls, config, roots, expander, data, pos):
def from_dict(cls, config, roots, data, pos):
keep_descriptions = config.yamlparser["keep_descriptions"]
d = data.copy()
name = d.pop_required_loc_string("name")
@ -34,7 +35,7 @@ class ViewBase(RootBase):
contents, params = split_contents_params(d, view_contents_keys)
return cls(
roots.defaults,
expander,
Expander(config),
keep_descriptions,
id,
name,
@ -58,8 +59,8 @@ class ViewBase(RootBase):
class View(ViewBase, NonTemplateRootMixin):
@classmethod
def add(cls, config, roots, expander, param_expander, data, pos):
view = cls.from_dict(config, roots, expander, data, pos)
def add(cls, config, roots, data, pos):
view = cls.from_dict(config, roots, data, pos)
roots.assign(roots.views, view.id, view, "view")
def __str__(self):
@ -68,8 +69,8 @@ class View(ViewBase, NonTemplateRootMixin):
class ViewTemplate(ViewBase, TemplateRootMixin):
@classmethod
def add(cls, config, roots, expander, params_expander, data, pos):
template = cls.from_dict(config, roots, params_expander, data, pos)
def add(cls, config, roots, data, pos):
template = cls.from_dict(config, roots, data, pos)
roots.assign(roots.view_templates, template.id, template, "view template")
def __str__(self):
@ -82,7 +83,7 @@ class ViewGroup(Group):
_view_templates: dict
@classmethod
def add(cls, config, roots, expander, params_expander, data, pos):
def add(cls, config, roots, data, pos):
d = data.copy()
name = d.pop_required_loc_string("name")
try:

View File

@ -273,12 +273,8 @@ class BaseYamlObject(metaclass=abc.ABCMeta):
@abc.abstractmethod
def expand(self, expander, params):
"""Expand object but do not substitute template parameters"""
pass
def subst(self, expander, params):
"""Expand object and substitute template parameters"""
return self.expand(expander, params)
pass
def _find_file(self, rel_path, pos):
search_path = self._search_path
@ -300,10 +296,6 @@ class BaseYamlObject(metaclass=abc.ABCMeta):
for idx, path in enumerate(path_list):
yield self._expand_path(path, path_list.value_pos[idx], *args)
def _subst_path_list(self, path_list, *args):
for idx, path in enumerate(path_list):
yield self._subst_path(path, path_list.value_pos[idx], *args)
class J2BaseYamlObject(BaseYamlObject):
def __init__(self, jjb_config, loader, pos):
@ -450,19 +442,11 @@ class IncludeRawBase(IncludeBaseObject):
def expand(self, expander, params):
return "\n".join(self._expand_path_list(self._path_list, params))
def subst(self, expander, params):
return "\n".join(self._subst_path_list(self._path_list, params))
class IncludeRaw(IncludeRawBase):
yaml_tag = "!include-raw:"
def _expand_path(self, rel_path_template, pos, params):
rel_path = self._formatter.format(rel_path_template, **params)
full_path = self._find_file(rel_path, pos)
return full_path.read_text()
def _subst_path(self, rel_path_template, pos, params):
rel_path = self._formatter.format(rel_path_template, **params)
full_path = self._find_file(rel_path, pos)
template = full_path.read_text()
@ -476,14 +460,6 @@ class IncludeRawEscape(IncludeRawBase):
yaml_tag = "!include-raw-escape:"
def _expand_path(self, rel_path_template, pos, params):
rel_path = self._formatter.format(rel_path_template, **params)
full_path = self._find_file(rel_path, pos)
text = full_path.read_text()
# Backward compatibility:
# if used inside job or macro without parameters, curly braces are duplicated.
return text.replace("{", "{{").replace("}", "}}")
def _subst_path(self, rel_path_template, pos, params):
rel_path = self._formatter.format(rel_path_template, **params)
full_path = self._find_file(rel_path, pos)
return full_path.read_text()

View File

@ -2,6 +2,6 @@
name: test-job-include-raw-1
builders:
- shell:
!include-raw include-raw001-hello-world.sh
!include-raw-escape include-raw001-hello-world.sh
- shell:
!include-raw include-raw001-vars.sh
!include-raw-escape include-raw001-vars.sh

View File

@ -6,7 +6,7 @@
"shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
},
{
"shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${{VAR1}} ${{VAR2}}\"\n\n[[ -n \"${{VAR3}}\" ]] && {{\n # this next section is executed as one\n echo \"${{VAR3}}\"\n exit 0\n}}\n"
"shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${VAR1} ${VAR2}\"\n\n[[ -n \"${VAR3}\" ]] && {\n # this next section is executed as one\n echo \"${VAR3}\"\n exit 0\n}\n"
}
],
"description": "<!-- Managed by Jenkins Job Builder -->"

View File

@ -30,13 +30,13 @@ exit 0
VAR1=&quot;hello&quot;
VAR2=&quot;world&quot;
VAR3=&quot;${{VAR1}} ${{VAR2}}&quot;
VAR3=&quot;${VAR1} ${VAR2}&quot;
[[ -n &quot;${{VAR3}}&quot; ]] &amp;&amp; {{
[[ -n &quot;${VAR3}&quot; ]] &amp;&amp; {
# this next section is executed as one
echo &quot;${{VAR3}}&quot;
echo &quot;${VAR3}&quot;
exit 0
}}
}
</command>
</hudson.tasks.Shell>
</builders>

View File

@ -1,4 +1,4 @@
# Using include-raw-excape inside job cause double braces in included file, like: {{VAR1}}.
# Using include-raw-escape inside job works the same way as inside job template.
- job:
name: test-job-include-raw
builders:

View File

@ -2,6 +2,6 @@
name: test-job-include-raw-1
builders:
- shell:
!include-raw:
!include-raw-escape:
- include-raw001-hello-world.sh
- include-raw001-vars.sh

View File

@ -2,6 +2,6 @@
name: test-job-include-raw-1
builders:
- shell:
!include-raw: include-raw001-hello-world.sh
!include-raw-escape: include-raw001-hello-world.sh
- shell:
!include-raw: include-raw001-vars.sh
!include-raw-escape: include-raw001-vars.sh

View File

@ -11,7 +11,7 @@
- xenial-{bdate}
jobs:
- 'template-requiring-param-{os}':
os: 'ubuntu-{flavour}'
os: 'ubuntu-{flavor}'
- job-template:
name: 'template-requiring-param-{os}'

View File

@ -0,0 +1,15 @@
macro_with_null_params.yaml:14:3: In project 'sample-project'
- project:
^
macro_with_null_params.yaml:17:9: Defined here
- sample-job
^
macro_with_null_params.yaml:8:3: In job template 'sample-job'
- job-template:
^
macro_with_null_params.yaml:1:3: While expanding macro 'sample-macro'
- builder:
^
macro_with_null_params.yaml:6:17: While formatting string 'echo {hello}': Missing parameter: 'hello'
- shell: 'echo {hello}'
^

View File

@ -2,13 +2,13 @@
name: sample-macro
builders:
# Add parameter to check if macro behaves the same way as if no params were provided.
# That is, it should not be expanded.
# That is, it should try to expand the parameter.
- shell: 'echo {hello}'
- job-template:
name: sample-job
builders:
# Place colon but define no parameters.
# Place colon but define no parameters (parameters are null instead of dict).
- sample-macro:
- project:

View File

@ -5,13 +5,13 @@ recursive_parameter.yaml:7:3: In job template 'sample-job-{param_1}'
- job-template:
^
recursive_parameter.yaml:5:9: Used by param_1
- '{param_2}-at-project'
- '{param_2}-at-globals'
^
recursive_parameter.yaml:9:14: Used by param_2
param_2: '{param_3}-at-template'
^
recursive_parameter.yaml:13:14: Used by param_3
param_3: '{param_4}-at-globals'
param_3: '{param_4}-at-project'
^
recursive_parameter.yaml:16:20: Used by param_4
param_4: '{param_1}-at-job-spec'
@ -20,5 +20,5 @@ recursive_parameter.yaml:3:5: While expanding 'param_1'
param_1:
^
recursive_parameter.yaml:5:9: Recursive parameters usage: param_1 <- param_2 <- param_3 <- param_4
- '{param_2}-at-project'
- '{param_2}-at-globals'
^

View File

@ -2,7 +2,7 @@
name: global
param_1:
- param_1_value_1
- '{param_2}-at-project'
- '{param_2}-at-globals'
- job-template:
name: 'sample-job-{param_1}'
@ -10,7 +10,7 @@
- project:
name: sample-project
param_3: '{param_4}-at-globals'
param_3: '{param_4}-at-project'
jobs:
- 'sample-job-{param_1}':
param_4: '{param_1}-at-job-spec'

View File

@ -9,8 +9,8 @@
name: builder-without-params
builders:
- shell: |
echo Should not be expanded: {param}
- shell: !include-raw: job-and-macro-expansions.yaml.no-expand.inc
echo Should not be expanded: {{param}}
- shell: !include-raw-escape: job-and-macro-expansions.yaml.no-expand.inc
- builder:
name: builder-with-params
@ -27,8 +27,8 @@
display-name: sample-job
builders:
- shell: |
echo Should not be expanded: {param}
- shell: !include-raw: job-and-macro-expansions.yaml.no-expand.inc
echo Should not be expanded: {{param}}
- shell: !include-raw-escape: job-and-macro-expansions.yaml.no-expand.inc
- job-template:
name: sample-job-template

View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders>
<hudson.tasks.Shell>
<command>echo {hello}</command>
</hudson.tasks.Shell>
</builders>
<publishers/>
<buildWrappers/>
</project>

View File

@ -4,6 +4,6 @@
dsl: |
build("job1")
parallel (
{ build("job2a") },
{ build("job2b") }
{{ build("job2a") }},
{{ build("job2b") }}
)

View File

@ -5,7 +5,7 @@
build job: "job1"
parallel [
2a: build job: "job2a",
2b: node "dummynode" {
2b: node "dummynode" {{
sh "echo I'm alive!"
}
}}
]

View File

@ -5,7 +5,7 @@
build job: "job1"
parallel [
2a: build job: "job2a",
2b: node "dummynode" {
2b: node "dummynode" {{
sh "echo I'm alive!"
}
}}
]

View File

@ -25,7 +25,7 @@
- timed: "H 14 * * *"
builders:
- shell: !include-raw: regression-2006254.inc
- shell: !include-raw-escape: regression-2006254.inc
parameters:
- bool:

View File

@ -13,7 +13,7 @@
<hudson.model.StringParameterDefinition>
<name>PARAM_1</name>
<description/>
<defaultValue>{default|my_default}</defaultValue>
<defaultValue>my_default</defaultValue>
<trim>false</trim>
</hudson.model.StringParameterDefinition>
<hudson.model.StringParameterDefinition>

View File

@ -1,5 +1,5 @@
# https://storyboard.openstack.org/#!/story/2010535
# Bug: JJB doesn't expand macro in case of usage without arguments
# Fixed: Bug: JJB doesn't expand macro in case of usage without arguments
# String templates in macro calls without parameters are NOT expanded.
# Jinja2 templates in macro calls without parameters ARE expanded.

View File

@ -12,7 +12,7 @@
publishers:
- trigger-parameterized-builds:
- project: first_job
predefined-parameters: BUILD_NUM=${BUILD_NUMBER}
predefined-parameters: BUILD_NUM=${{BUILD_NUMBER}}
property-file: default_version.prop
current-parameters: true
- project: second_job
@ -25,7 +25,7 @@
publishers:
- trigger-parameterized-builds:
- project: 1.2_first_job
predefined-parameters: BUILD_NUM=${BUILD_NUMBER}
predefined-parameters: BUILD_NUM=${{BUILD_NUMBER}}
current-parameters: true
property-file: version.prop
- project: 1.2_second_job