Support excludes when recursively processing paths
Include an additional option to exclude some paths/patterns from being included in the search to allow a complex hierarchy of directories to be referenced under a single path. Provide support to allow the user to specify patterns, relative and/or absolute path references. Implements support for excluding based on absolute path, relative path and based on simple shell globing patterns. Change-Id: I236b364c268fd2bf5511a2c6d9a1c87914f3b086
This commit is contained in:
Darragh Bailey
parent
74c8ac4561
commit
c5c0eb4f16
|
|
@ -74,6 +74,14 @@ job_builder section
|
|||
(Optional) If set to True, jenkins job builder will search for job
|
||||
definition files recursively
|
||||
|
||||
**exclude**
|
||||
(Optional) If set to a list of values separated by ':', these paths will be
|
||||
excluded from the list of paths to be processed when searching recursively.
|
||||
Values containing no ``/`` will be matched against directory names at all
|
||||
levels, those starting with ``/`` will be considered absolute, while others
|
||||
containing a ``/`` somewhere other than the start of the value will be
|
||||
considered relative to the starting path.
|
||||
|
||||
**allow_duplicates**
|
||||
(Optional) By default `jenkins-jobs` will abort any time a duplicate macro,
|
||||
template, job-group or job name is encountered as it cannot establish the
|
||||
|
|
@ -155,6 +163,59 @@ Jenkins instances suited to various needs you may want to share configuration
|
|||
between those instances (global). Furthermore, there may be various ways you
|
||||
would like to structure jobs within a given instance.
|
||||
|
||||
Recursive Searching of Paths
|
||||
----------------------------
|
||||
|
||||
In addition to passing multiple paths to JJB it is also possible to enable
|
||||
recursive searching to process all yaml files in the tree beneath each path.
|
||||
For example::
|
||||
|
||||
For a tree:
|
||||
/path/
|
||||
to/
|
||||
defs/
|
||||
ci_jobs/
|
||||
release_jobs/
|
||||
globals/
|
||||
macros/
|
||||
templates/
|
||||
|
||||
jenkins-jobs update -r /path/to/defs:/path/to/globals
|
||||
|
||||
JJB will search defs/ci_jobs, defs/release_jobs, globals/macros and
|
||||
globals/templates in addition to the defs and globals trees.
|
||||
|
||||
Excluding Paths
|
||||
---------------
|
||||
|
||||
To allow a complex tree of jobs where some jobs are managed differently without
|
||||
needing to explicitly provide each path, the recursive path processing supports
|
||||
excluding paths based on absolute paths, relative paths and patterns. For
|
||||
example::
|
||||
|
||||
For a tree:
|
||||
/path/
|
||||
to/
|
||||
defs/
|
||||
ci_jobs/
|
||||
manual/
|
||||
release_jobs/
|
||||
manual/
|
||||
qa_jobs/
|
||||
globals/
|
||||
macros/
|
||||
templates/
|
||||
special/
|
||||
|
||||
jenkins-jobs update -r -x man*:./qa_jobs -x /path/to/defs/globals/special \
|
||||
/path/to/defs:/path/to/globals
|
||||
|
||||
JJB search the given paths, ignoring the directories qa_jobs, ci_jobs/manual,
|
||||
release_jobs/manual, and globals/special when building the list of yaml files
|
||||
to be processed. Absolute paths are denoted by starting from the root,
|
||||
relative by containing the path separator, and patterns by having neither.
|
||||
Patterns use simple shell globing to match directories.
|
||||
|
||||
Deleting Jobs
|
||||
^^^^^^^^^^^^^
|
||||
Jenkins Job Builder supports deleting jobs from Jenkins.
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ ignore_cache=True
|
|||
keep_descriptions=False
|
||||
include_path=.:scripts:~/git/
|
||||
recursive=False
|
||||
exclude=.*:manual:./development
|
||||
allow_duplicates=False
|
||||
|
||||
[jenkins]
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
import argparse
|
||||
from six.moves import configparser, StringIO
|
||||
import fnmatch
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
|
|
@ -33,6 +34,7 @@ DEFAULT_CONF = """
|
|||
keep_descriptions=False
|
||||
ignore_cache=False
|
||||
recursive=False
|
||||
exclude=.*
|
||||
allow_duplicates=False
|
||||
|
||||
[jenkins]
|
||||
|
|
@ -51,11 +53,28 @@ def confirm(question):
|
|||
sys.exit('Aborted')
|
||||
|
||||
|
||||
def recurse_path(root):
|
||||
def recurse_path(root, excludes=None):
|
||||
if excludes is None:
|
||||
excludes = []
|
||||
|
||||
basepath = os.path.realpath(root)
|
||||
pathlist = [basepath]
|
||||
|
||||
patterns = [e for e in excludes if os.path.sep not in e]
|
||||
absolute = [e for e in excludes if os.path.isabs(e)]
|
||||
relative = [e for e in excludes if os.path.sep in e and
|
||||
not os.path.isabs(e)]
|
||||
for root, dirs, files in os.walk(basepath, topdown=True):
|
||||
dirs[:] = [
|
||||
d for d in dirs
|
||||
if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns])
|
||||
if not any([fnmatch.fnmatch(os.path.abspath(os.path.join(root, d)),
|
||||
path)
|
||||
for path in absolute])
|
||||
if not any([fnmatch.fnmatch(os.path.relpath(os.path.join(root, d)),
|
||||
path)
|
||||
for path in relative])
|
||||
]
|
||||
pathlist.extend([os.path.join(root, path) for path in dirs])
|
||||
|
||||
return pathlist
|
||||
|
|
@ -68,6 +87,10 @@ def create_parser():
|
|||
recursive_parser.add_argument('-r', '--recursive', action='store_true',
|
||||
dest='recursive', default=False,
|
||||
help='look for yaml files recursively')
|
||||
recursive_parser.add_argument('-x', '--exclude', dest='exclude',
|
||||
action='append', default=[],
|
||||
help='paths to exclude when using recursive'
|
||||
' search, uses standard globbing.')
|
||||
subparser = parser.add_subparsers(help='update, test or delete job',
|
||||
dest='command')
|
||||
|
||||
|
|
@ -95,7 +118,7 @@ def create_parser():
|
|||
|
||||
# subparser: delete
|
||||
|
||||
parser_delete = subparser.add_parser('delete')
|
||||
parser_delete = subparser.add_parser('delete', parents=[recursive_parser])
|
||||
parser_delete.add_argument('name', help='name of job', nargs='+')
|
||||
parser_delete.add_argument('-p', '--path', default=None,
|
||||
help='colon-separated list of paths to'
|
||||
|
|
@ -233,10 +256,13 @@ def execute(options, config):
|
|||
do_recurse = (getattr(options, 'recursive', False) or
|
||||
config.getboolean('job_builder', 'recursive'))
|
||||
|
||||
excludes = [e for elist in options.exclude
|
||||
for e in elist.split(os.pathsep)] or \
|
||||
config.get('job_builder', 'exclude').split(os.pathsep)
|
||||
paths = []
|
||||
for path in options.path:
|
||||
if do_recurse and os.path.isdir(path):
|
||||
paths.extend(recurse_path(path))
|
||||
paths.extend(recurse_path(path, excludes))
|
||||
else:
|
||||
paths.append(path)
|
||||
options.path = paths
|
||||
|
|
|
|||
|
|
@ -7,40 +7,41 @@ import jenkins
|
|||
|
||||
from jenkins_jobs import cmd
|
||||
from jenkins_jobs.errors import JenkinsJobsException
|
||||
from tests.cmd.test_cmd import CmdTestsBase
|
||||
from tests.base import mock
|
||||
from tests.cmd.test_cmd import CmdTestsBase
|
||||
from tests.cmd.test_recurse_path import fake_os_walk
|
||||
|
||||
|
||||
os_walk_return_values = {
|
||||
'/jjb_projects': [
|
||||
('/jjb_projects', ('dir1', 'dir2', 'dir3'), ()),
|
||||
('/jjb_projects/dir1', ('bar',), ()),
|
||||
('/jjb_projects/dir2', ('baz',), ()),
|
||||
('/jjb_projects/dir3', (), ()),
|
||||
('/jjb_projects/dir1/bar', (), ()),
|
||||
('/jjb_projects/dir2/baz', (), ()),
|
||||
('/jjb_projects', (['dir1', 'dir2', 'dir3'], ())),
|
||||
('/jjb_projects/dir1', (['bar'], ())),
|
||||
('/jjb_projects/dir2', (['baz'], ())),
|
||||
('/jjb_projects/dir3', ([], ())),
|
||||
('/jjb_projects/dir1/bar', ([], ())),
|
||||
('/jjb_projects/dir2/baz', ([], ())),
|
||||
],
|
||||
'/jjb_templates': [
|
||||
('/jjb_templates', ('dir1', 'dir2', 'dir3'), ()),
|
||||
('/jjb_templates/dir1', ('bar',), ()),
|
||||
('/jjb_templates/dir2', ('baz',), ()),
|
||||
('/jjb_templates/dir3', (), ()),
|
||||
('/jjb_templates/dir1/bar', (), ()),
|
||||
('/jjb_templates/dir2/baz', (), ()),
|
||||
('/jjb_templates', (['dir1', 'dir2', 'dir3'], ())),
|
||||
('/jjb_templates/dir1', (['bar'], ())),
|
||||
('/jjb_templates/dir2', (['baz'], ())),
|
||||
('/jjb_templates/dir3', ([], ())),
|
||||
('/jjb_templates/dir1/bar', ([], ())),
|
||||
('/jjb_templates/dir2/baz', ([], ())),
|
||||
],
|
||||
'/jjb_macros': [
|
||||
('/jjb_macros', ('dir1', 'dir2', 'dir3'), ()),
|
||||
('/jjb_macros/dir1', ('bar',), ()),
|
||||
('/jjb_macros/dir2', ('baz',), ()),
|
||||
('/jjb_macros/dir3', (), ()),
|
||||
('/jjb_macros/dir1/bar', (), ()),
|
||||
('/jjb_macros/dir2/baz', (), ()),
|
||||
('/jjb_macros', (['dir1', 'dir2', 'dir3'], ())),
|
||||
('/jjb_macros/dir1', (['bar'], ())),
|
||||
('/jjb_macros/dir2', (['baz'], ())),
|
||||
('/jjb_macros/dir3', ([], ())),
|
||||
('/jjb_macros/dir1/bar', ([], ())),
|
||||
('/jjb_macros/dir2/baz', ([], ())),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def os_walk_side_effects(path_name, topdown):
|
||||
return os_walk_return_values[path_name]
|
||||
return fake_os_walk(os_walk_return_values[path_name])(path_name, topdown)
|
||||
|
||||
|
||||
@mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock)
|
||||
|
|
@ -114,8 +115,7 @@ class TestTests(CmdTestsBase):
|
|||
path_list = os_walk_return_values.keys()
|
||||
paths = []
|
||||
for path in path_list:
|
||||
paths.extend([p for p, _, _ in
|
||||
os_walk_return_values[path]])
|
||||
paths.extend([p for p, _ in os_walk_return_values[path]])
|
||||
|
||||
multipath = os.pathsep.join(path_list)
|
||||
|
||||
|
|
@ -132,6 +132,34 @@ class TestTests(CmdTestsBase):
|
|||
|
||||
update_job_mock.assert_called_with(paths, [], output=args.output_dir)
|
||||
|
||||
@mock.patch('jenkins_jobs.cmd.Builder.update_job')
|
||||
@mock.patch('jenkins_jobs.cmd.os.path.isdir')
|
||||
@mock.patch('jenkins_jobs.cmd.os.walk')
|
||||
def test_recursive_multi_path_with_excludes(self, os_walk_mock, isdir_mock,
|
||||
update_job_mock):
|
||||
"""
|
||||
Run test mode and pass multiple paths with recursive path option.
|
||||
"""
|
||||
|
||||
os_walk_mock.side_effect = os_walk_side_effects
|
||||
isdir_mock.return_value = True
|
||||
|
||||
path_list = os_walk_return_values.keys()
|
||||
paths = []
|
||||
for path in path_list:
|
||||
paths.extend([p for p, __ in os_walk_return_values[path]
|
||||
if 'dir1' not in p and 'dir2' not in p])
|
||||
|
||||
multipath = os.pathsep.join(path_list)
|
||||
|
||||
args = self.parser.parse_args(['test', '-r', multipath, '-x',
|
||||
'dir1:dir2'])
|
||||
args.output_dir = mock.MagicMock()
|
||||
|
||||
cmd.execute(args, self.config)
|
||||
|
||||
update_job_mock.assert_called_with(paths, [], output=args.output_dir)
|
||||
|
||||
def test_console_output(self):
|
||||
"""
|
||||
Run test mode and verify that resulting XML gets sent to the console.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
import os
|
||||
|
||||
import mock
|
||||
import testtools
|
||||
|
||||
from jenkins_jobs import cmd
|
||||
|
||||
|
||||
def fake_os_walk(paths):
|
||||
"""Helper function for mocking os.walk() where must test that manipulation
|
||||
of the returned dirs variable works as expected
|
||||
"""
|
||||
paths_dict = dict(paths)
|
||||
|
||||
def os_walk(top, topdown=True):
|
||||
dirs, nondirs = paths_dict[top]
|
||||
yield top, dirs, nondirs
|
||||
for name in dirs:
|
||||
# hard code use of '/' to ensure the test data can be defined as
|
||||
# simple strings otherwise tests using this helper will break on
|
||||
# platforms where os.path.sep is different.
|
||||
new_path = "/".join([top, name])
|
||||
for x in os_walk(new_path, topdown):
|
||||
yield x
|
||||
return os_walk
|
||||
|
||||
|
||||
# Testing the cmd module can sometimes result in the CacheStorage class
|
||||
# attempting to create the cache directory multiple times as the tests
|
||||
# are run in parallel. Stub out the CacheStorage to ensure that each
|
||||
# test can safely create the object without effect.
|
||||
@mock.patch('jenkins_jobs.builder.CacheStorage', mock.MagicMock)
|
||||
class CmdRecursePath(testtools.TestCase):
|
||||
|
||||
@mock.patch('jenkins_jobs.cmd.os.walk')
|
||||
def test_recursive_path_option_exclude_pattern(self, oswalk_mock):
|
||||
"""
|
||||
Test paths returned by the recursive processing when using pattern
|
||||
excludes.
|
||||
|
||||
testing paths
|
||||
/jjb_configs/dir1/test1/
|
||||
/jjb_configs/dir1/file
|
||||
/jjb_configs/dir2/test2/
|
||||
/jjb_configs/dir3/bar/
|
||||
/jjb_configs/test3/bar/
|
||||
/jjb_configs/test3/baz/
|
||||
"""
|
||||
|
||||
os_walk_paths = [
|
||||
('/jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
|
||||
('/jjb_configs/dir1', (['test1'], ('file'))),
|
||||
('/jjb_configs/dir2', (['test2'], ())),
|
||||
('/jjb_configs/dir3', (['bar'], ())),
|
||||
('/jjb_configs/dir3/bar', ([], ())),
|
||||
('/jjb_configs/test3/bar', None),
|
||||
('/jjb_configs/test3/baz', None)
|
||||
]
|
||||
|
||||
paths = [k for k, v in os_walk_paths if v is not None]
|
||||
|
||||
oswalk_mock.side_effect = fake_os_walk(os_walk_paths)
|
||||
self.assertEqual(paths, cmd.recurse_path('/jjb_configs', ['test*']))
|
||||
|
||||
@mock.patch('jenkins_jobs.cmd.os.walk')
|
||||
def test_recursive_path_option_exclude_absolute(self, oswalk_mock):
|
||||
"""
|
||||
Test paths returned by the recursive processing when using absolute
|
||||
excludes.
|
||||
|
||||
testing paths
|
||||
/jjb_configs/dir1/test1/
|
||||
/jjb_configs/dir1/file
|
||||
/jjb_configs/dir2/test2/
|
||||
/jjb_configs/dir3/bar/
|
||||
/jjb_configs/test3/bar/
|
||||
/jjb_configs/test3/baz/
|
||||
"""
|
||||
|
||||
os_walk_paths = [
|
||||
('/jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
|
||||
('/jjb_configs/dir1', None),
|
||||
('/jjb_configs/dir2', (['test2'], ())),
|
||||
('/jjb_configs/dir3', (['bar'], ())),
|
||||
('/jjb_configs/test3', (['bar', 'baz'], ())),
|
||||
('/jjb_configs/dir2/test2', ([], ())),
|
||||
('/jjb_configs/dir3/bar', ([], ())),
|
||||
('/jjb_configs/test3/bar', ([], ())),
|
||||
('/jjb_configs/test3/baz', ([], ()))
|
||||
]
|
||||
|
||||
paths = [k for k, v in os_walk_paths if v is not None]
|
||||
|
||||
oswalk_mock.side_effect = fake_os_walk(os_walk_paths)
|
||||
|
||||
self.assertEqual(paths, cmd.recurse_path('/jjb_configs',
|
||||
['/jjb_configs/dir1']))
|
||||
|
||||
@mock.patch('jenkins_jobs.cmd.os.walk')
|
||||
def test_recursive_path_option_exclude_relative(self, oswalk_mock):
|
||||
"""
|
||||
Test paths returned by the recursive processing when using relative
|
||||
excludes.
|
||||
|
||||
testing paths
|
||||
./jjb_configs/dir1/test/
|
||||
./jjb_configs/dir1/file
|
||||
./jjb_configs/dir2/test/
|
||||
./jjb_configs/dir3/bar/
|
||||
./jjb_configs/test3/bar/
|
||||
./jjb_configs/test3/baz/
|
||||
"""
|
||||
|
||||
os_walk_paths = [
|
||||
('jjb_configs', (['dir1', 'dir2', 'dir3', 'test3'], ())),
|
||||
('jjb_configs/dir1', (['test'], ('file'))),
|
||||
('jjb_configs/dir2', (['test2'], ())),
|
||||
('jjb_configs/dir3', (['bar'], ())),
|
||||
('jjb_configs/test3', (['bar', 'baz'], ())),
|
||||
('jjb_configs/dir1/test', ([], ())),
|
||||
('jjb_configs/dir2/test2', ([], ())),
|
||||
('jjb_configs/dir3/bar', ([], ())),
|
||||
('jjb_configs/test3/bar', None),
|
||||
('jjb_configs/test3/baz', ([], ()))
|
||||
]
|
||||
|
||||
rel_os_walk_paths = [
|
||||
(os.path.abspath(
|
||||
os.path.join(os.path.curdir, k)), v) for k, v in os_walk_paths]
|
||||
|
||||
paths = [k for k, v in rel_os_walk_paths if v is not None]
|
||||
|
||||
oswalk_mock.side_effect = fake_os_walk(rel_os_walk_paths)
|
||||
|
||||
self.assertEqual(paths, cmd.recurse_path('jjb_configs',
|
||||
['jjb_configs/test3/bar']))
|
||||
Loading…
Reference in New Issue