Introduce modular implementation of subcommands.

This commit intentionally introduces a number of important API
breakages. Specifically, the jenkins_jobs.cmd module has been pared
down to some of its most difficult-to-refactor elements.

* Create jenkins_jobs.cli.entry.JenkinsJobs class to organize command
  line parsing and execution.
* Remove references to ConfigParser object in test code, hidden as an
  implementation detail of JenkinsJobs command line parsing. This will
  be necessary in the next stage of JJB 2.0 code which will be to
  create a JJBConfig object that handles logic and presentation of
  configuration from various sources--defaults, command line
  arguments, configuration file, and maybe environment variables in
  the future.
* Remove references to Namespace object produced by argparse module.
  Required rewrite of multipath & recursive path tests with a new
  MatchesDir testtools Matcher class that validates the expected
  output for a run of JJB against a given set of yamldirs with the
  specified command line arguments.
* Use stevedore to dynamically load subcommand parsers.
* Move configuration loading/testing to its own test file. Also fix
  the global vs home directory JJB config file test.

Change-Id: If62280418ba7319c313033ab387af4284237747e
changes/52/180252/11
Wayne 8 years ago committed by Wayne Warren
parent 1c22158672
commit fc73cedb45

@ -0,0 +1,139 @@
#!/usr/bin/env python
# Copyright (C) 2015 Wayne Warren
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import logging
import sys
from stevedore import extension
import jenkins_jobs.version
from jenkins_jobs import cmd
logging.basicConfig(level=logging.INFO)
def __version__():
return "Jenkins Job Builder version: %s" % \
jenkins_jobs.version.version_info.version_string()
class JenkinsJobs(object):
""" This is the entry point class for the `jenkins-jobs` command line tool.
While this class can be used programmatically by external users of the JJB
API, the main goal here is to abstract the `jenkins_jobs` tool in a way
that prevents test suites from caring overly much about various
implementation details--for example, tests of subcommands must not have
access to directly modify configuration objects, instead they must provide
a fixture in the form of an .ini file that provides the configuration
necessary for testing.
External users of the JJB API may be interested in this class as an
alternative to wrapping `jenkins_jobs` with a subprocess that execs it as a
system command; instead, python scripts may be written that pass
`jenkins_jobs` args directly to this class to allow programmatic setting of
various command line parameters.
"""
def __init__(self, args=None):
if args is None:
args = []
parser = self._create_parser()
self._options = parser.parse_args(args)
if not self._options.command:
parser.error("Must specify a 'command' to be performed")
logger = logging.getLogger()
if (self._options.log_level is not None):
self._options.log_level = getattr(logging,
self._options.log_level.upper(),
logger.getEffectiveLevel())
logger.setLevel(self._options.log_level)
self._config_file_values = cmd.setup_config_settings(self._options)
def _create_parser(self):
parser = argparse.ArgumentParser()
parser.add_argument(
'--conf',
dest='conf',
help='''configuration file''')
parser.add_argument(
'-l',
'--log_level',
dest='log_level',
default='info',
help='''log level (default: %(default)s)''')
parser.add_argument(
'--ignore-cache',
action='store_true',
dest='ignore_cache',
default=False,
help='''ignore the cache and update the jobs anyhow (that will only
flush the specified jobs cache)''')
parser.add_argument(
'--flush-cache',
action='store_true',
dest='flush_cache',
default=False,
help='''flush all the cache entries before updating''')
parser.add_argument(
'--version',
dest='version',
action='version',
version=__version__(),
help='''show version''')
parser.add_argument(
'--allow-empty-variables',
action='store_true',
dest='allow_empty_variables',
default=None,
help='''Don\'t fail if any of the variables inside any string are
not defined, replace with empty string instead.''')
parser.add_argument(
'--user', '-u',
help='''The Jenkins user to use for authentication. This overrides
the user specified in the configuration file.''')
parser.add_argument(
'--password', '-p',
help='''Password or API token to use for authenticating towards
Jenkins. This overrides the password specified in the configuration
file.''')
subparser = parser.add_subparsers(
dest='command',
help='''update, test or delete job''')
extension_manager = extension.ExtensionManager(
namespace='jjb.cli.subcommands',
invoke_on_load=True,
)
def parse_subcommand_args(ext, subparser):
ext.obj.parse_args(subparser)
extension_manager.map(parse_subcommand_args, subparser)
return parser
def execute(self):
jenkins_jobs.cmd.execute(self._options, self._config_file_values)
def main():
argv = sys.argv[1:]
jjb = JenkinsJobs(argv)
jjb.execute()

@ -0,0 +1,67 @@
#!/usr/bin/env python
# Copyright (C) 2015 Wayne Warren
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class BaseSubCommand(object):
"""Base class for Jenkins Job Builder subcommands, intended to allow
subcommands to be loaded as stevedore extensions by third party users.
"""
def __init__(self):
pass
@abc.abstractmethod
def parse_args(self, subparsers, recursive_parser):
"""Define subcommand arguments.
:param subparsers
A sub parser object. Implementations of this method should
create a new subcommand parser by calling
parser = subparsers.add_parser('command-name', ...)
This will return a new ArgumentParser object; all other arguments to
this method will be passed to the argparse.ArgumentParser constructor
for the returned object.
"""
@abc.abstractmethod
def execute(self, config):
"""Execute subcommand behavior.
:param config
JJBConfig object containing final configuration from config files,
command line arguments, and environment variables.
"""
@staticmethod
def parse_option_recursive_exclude(parser):
"""Add '--recursive' and '--exclude' arguments to given parser.
"""
parser.add_argument(
'-r', '--recursive',
action='store_true',
dest='recursive',
default=False,
help='''look for yaml files recursively''')
parser.add_argument(
'-x', '--exclude',
dest='exclude',
action='append',
default=[],
help='''paths to exclude when using recursive search, uses standard
globbing.''')

@ -0,0 +1,23 @@
import jenkins_jobs.cli.subcommand.base as base
class DeleteSubCommand(base.BaseSubCommand):
def parse_args(self, subparser):
delete = subparser.add_parser('delete')
self.parse_option_recursive_exclude(delete)
delete.add_argument(
'name',
help='name of job',
nargs='+')
delete.add_argument(
'-p', '--path',
default=None,
help='''colon-separated list of paths to YAML files or
directories''')
def execute(self, config):
raise NotImplementedError

@ -0,0 +1,16 @@
import jenkins_jobs.cli.subcommand.base as base
class DeleteAllSubCommand(base.BaseSubCommand):
def parse_args(self, subparser):
delete_all = subparser.add_parser(
'delete-all',
help='''delete *ALL* jobs from Jenkins server, including those not
managed by Jenkins Job Builder.''')
self.parse_option_recursive_exclude(delete_all)
def execute(self, config):
raise NotImplementedError

@ -0,0 +1,33 @@
import sys
import jenkins_jobs.cli.subcommand.base as base
class TestSubCommand(base.BaseSubCommand):
def parse_args(self, subparser):
test = subparser.add_parser('test')
self.parse_option_recursive_exclude(test)
test.add_argument(
'path',
help='''colon-separated list of paths to YAML files or
directories''',
nargs='?',
default=sys.stdin)
test.add_argument(
'-p',
dest='plugins_info_path',
default=None,
help='path to plugin info YAML file')
test.add_argument(
'-o',
dest='output_dir',
default=sys.stdout,
help='path to output XML')
test.add_argument(
'name',
help='name(s) of job(s)', nargs='*')
def execute(self, config):
raise NotImplementedError

@ -0,0 +1,33 @@
import jenkins_jobs.cli.subcommand.base as base
class UpdateSubCommand(base.BaseSubCommand):
def parse_args(self, subparser):
update = subparser.add_parser('update')
self.parse_option_recursive_exclude(update)
update.add_argument(
'path',
help='''colon-separated list of paths to YAML files or
directories''')
update.add_argument(
'names',
help='name(s) of job(s)', nargs='*')
update.add_argument(
'--delete-old',
action='store_true',
dest='delete_old',
default=False,
help='delete obsolete jobs')
update.add_argument(
'--workers',
type=int,
default=1,
dest='n_workers',
help='''number of workers to use, 0 for autodetection and 1 for
just one worker.''')
def execute(self, config):
raise NotImplementedError

@ -13,7 +13,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import fnmatch
import io
import logging
@ -31,6 +30,8 @@ from jenkins_jobs.errors import JenkinsJobsException
import jenkins_jobs.version
import jenkins_jobs.cli
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
@ -89,110 +90,7 @@ def recurse_path(root, excludes=None):
return pathlist
def create_parser():
parser = argparse.ArgumentParser()
recursive_parser = argparse.ArgumentParser(add_help=False)
recursive_parser.add_argument('-r', '--recursive', action='store_true',
dest='recursive', default=False,
help='look for yaml files recursively')
recursive_parser.add_argument('-x', '--exclude', dest='exclude',
action='append', default=[],
help='paths to exclude when using recursive'
' search, uses standard globbing.')
subparser = parser.add_subparsers(help='update, test or delete job',
dest='command')
# subparser: update
parser_update = subparser.add_parser('update', parents=[recursive_parser])
parser_update.add_argument('path', help='colon-separated list of paths to'
' YAML files or directories')
parser_update.add_argument('names', help='name(s) of job(s)', nargs='*')
parser_update.add_argument('--delete-old', help='delete obsolete jobs',
action='store_true',
dest='delete_old', default=False,)
parser_update.add_argument('--workers', dest='n_workers', type=int,
default=1, help='number of workers to use, 0 '
'for autodetection and 1 for just one worker.')
# subparser: test
parser_test = subparser.add_parser('test', parents=[recursive_parser])
parser_test.add_argument('path', help='colon-separated list of paths to'
' YAML files or directories',
nargs='?', default=sys.stdin)
parser_test.add_argument('-p', dest='plugins_info_path', default=None,
help='path to plugin info YAML file')
parser_test.add_argument('-o', dest='output_dir', default=sys.stdout,
help='path to output XML')
parser_test.add_argument('name', help='name(s) of job(s)', nargs='*')
# subparser: delete
parser_delete = subparser.add_parser('delete', parents=[recursive_parser])
parser_delete.add_argument('name', help='name of job', nargs='+')
parser_delete.add_argument('-p', '--path', default=None,
help='colon-separated list of paths to'
' YAML files or directories')
# subparser: delete-all
subparser.add_parser('delete-all',
help='delete *ALL* jobs from Jenkins server, '
'including those not managed by Jenkins Job '
'Builder.')
parser.add_argument('--conf', dest='conf', help='configuration file')
parser.add_argument('-l', '--log_level', dest='log_level', default='info',
help="log level (default: %(default)s)")
parser.add_argument(
'--ignore-cache', action='store_true',
dest='ignore_cache', default=False,
help='ignore the cache and update the jobs anyhow (that will only '
'flush the specified jobs cache)')
parser.add_argument(
'--flush-cache', action='store_true', dest='flush_cache',
default=False, help='flush all the cache entries before updating')
parser.add_argument('--version', dest='version', action='version',
version=version(),
help='show version')
parser.add_argument(
'--allow-empty-variables', action='store_true',
dest='allow_empty_variables', default=None,
help='Don\'t fail if any of the variables inside any string are not '
'defined, replace with empty string instead')
parser.add_argument(
'--user', '-u',
help='The Jenkins user to use for authentication. This overrides '
'the user specified in the configuration file')
parser.add_argument(
'--password', '-p',
help='Password or API token to use for authenticating towards '
'Jenkins. This overrides the password specified in the '
'configuration file.')
return parser
def main(argv=None):
# We default argv to None and assign to sys.argv[1:] below because having
# an argument default value be a mutable type in Python is a gotcha. See
# http://bit.ly/1o18Vff
if argv is None:
argv = sys.argv[1:]
parser = create_parser()
options = parser.parse_args(argv)
if not options.command:
parser.error("Must specify a 'command' to be performed")
if (options.log_level is not None):
options.log_level = getattr(logging, options.log_level.upper(),
logger.getEffectiveLevel())
logger.setLevel(options.log_level)
config = setup_config_settings(options)
execute(options, config)
def get_config_file(options):
# Initialize with the global fallback location for the config.
conf = '/etc/jenkins_jobs/jenkins_jobs.ini'
if options.conf:
conf = options.conf
@ -211,7 +109,6 @@ def get_config_file(options):
def setup_config_settings(options):
conf = get_config_file(options)
config = configparser.ConfigParser()
# Load default config always
@ -225,8 +122,8 @@ def setup_config_settings(options):
logger.debug("Not requiring config for test output generation")
else:
raise JenkinsJobsException(
"A valid configuration file is required when not run as a test"
"\n{0} is not a valid .ini file".format(conf))
"A valid configuration file is required."
"\n{0} is not valid.".format(conf))
return config
@ -378,13 +275,3 @@ def execute(options, config):
builder.update_jobs(options.path, options.name,
output=options.output_dir,
n_workers=1)
def version():
return "Jenkins Job Builder version: %s" % \
jenkins_jobs.version.version_info.version_string()
if __name__ == '__main__':
sys.path.insert(0, '.')
main()

@ -2,3 +2,4 @@ six>=1.5.2
PyYAML
python-jenkins>=0.4.8
pbr>=1.0.0,<2.0
stevedore==1.8.0

@ -34,7 +34,12 @@ warnerrors = True
[entry_points]
console_scripts =
jenkins-jobs=jenkins_jobs.cmd:main
jenkins-jobs=jenkins_jobs.cli.entry:main
jjb.cli.subcommands =
update=jenkins_jobs.cli.subcommand.update:UpdateSubCommand
test=jenkins_jobs.cli.subcommand.test:TestSubCommand
delete=jenkins_jobs.cli.subcommand.delete:DeleteSubCommand
delet-all=jenkins_jobs.cli.subcommand.delete_all:DeleteAllSubCommand
jenkins_jobs.projects =
externaljob=jenkins_jobs.modules.project_externaljob:ExternalJob
flow=jenkins_jobs.modules.project_flow:Flow

@ -0,0 +1,2 @@
[job_builder]
recursive=True

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<disabled>true</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<disabled>true</disabled>
<displayName>herp derp derp</displayName>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<matrix-project>
<executionStrategy class="hudson.matrix.DefaultMatrixExecutionStrategyImpl">
<runSequentially>false</runSequentially>
</executionStrategy>
<combinationFilter/>
<axes/>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</matrix-project>

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<disabled>true</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<disabled>true</disabled>
<displayName>herp derp derp</displayName>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<disabled>true</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<disabled>true</disabled>
<displayName>herp derp derp</displayName>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders/>
<publishers/>
<buildWrappers/>
</project>

@ -0,0 +1,7 @@
- project:
name: 'hello'
jobs:
- 'job1'
- job:
name: 'job1'

@ -0,0 +1,8 @@
- project:
name: 'beep'
jobs:
- 'job2'
- job:
name: 'job2'
disabled: True

@ -0,0 +1,8 @@
- project:
name: 'goodbye'
jobs:
- 'job4'
- job:
name: 'job4'
project-type: matrix

@ -0,0 +1,9 @@
- project:
name: 'meow'
jobs:
- 'job3'
- job:
name: 'job3'
display-name: 'herp derp derp'
disabled: True

@ -1,6 +1,5 @@
import os
from jenkins_jobs import cmd
from tests.base import mock
from tests.cmd.test_cmd import CmdTestsBase
@ -14,8 +13,8 @@ class DeleteTests(CmdTestsBase):
Test handling the deletion of a single Jenkins job.
"""
args = self.parser.parse_args(['delete', 'test_job'])
cmd.execute(args, self.config) # passes if executed without error
args = ['--conf', self.default_config_file, 'delete', 'test_job']
self.execute_jenkins_jobs_with_args(args)
@mock.patch('jenkins_jobs.cmd.Builder.delete_job')
def test_delete_multiple_jobs(self, delete_job_mock):
@ -23,8 +22,9 @@ class DeleteTests(CmdTestsBase):
Test handling the deletion of multiple Jenkins jobs.
"""
args = self.parser.parse_args(['delete', 'test_job1', 'test_job2'])
cmd.execute(args, self.config) # passes if executed without error
args = ['--conf', self.default_config_file,
'delete', 'test_job1', 'test_job2']
self.execute_jenkins_jobs_with_args(args)
@mock.patch('jenkins_jobs.builder.Jenkins.delete_job')
def test_delete_using_glob_params(self, delete_job_mock):
@ -33,12 +33,12 @@ class DeleteTests(CmdTestsBase):
parameters feature.
"""
args = self.parser.parse_args(['delete',
'--path',
os.path.join(self.fixtures_path,
'cmd-002.yaml'),
'*bar*'])
cmd.execute(args, self.config)
args = ['--conf', self.default_config_file,
'delete', '--path',
os.path.join(self.fixtures_path,
'cmd-002.yaml'),
'*bar*']
self.execute_jenkins_jobs_with_args(args)
calls = [mock.call('bar001'), mock.call('bar002')]
delete_job_mock.assert_has_calls(calls, any_order=True)
self.assertEqual(delete_job_mock.call_count, len(calls),

@ -1,194 +1,43 @@
import difflib
import filecmp
import io
import os
import shutil
import tempfile
import yaml
import jenkins
import testtools
from jenkins_jobs import cmd
from jenkins_jobs.cli import entry
from jenkins_jobs.errors import JenkinsJobsException
from mock import patch
from tests.base import mock
from tests.cmd.test_cmd import CmdTestsBase
from tests.cmd.test_recurse_path import fake_os_walk
os_walk_return_values = {
'/jjb_projects': [
('/jjb_projects', (['dir1', 'dir2', 'dir3'], ())),
('/jjb_projects/dir1', (['bar'], ())),
('/jjb_projects/dir2', (['baz'], ())),
('/jjb_projects/dir3', ([], ())),
('/jjb_projects/dir1/bar', ([], ())),
('/jjb_projects/dir2/baz', ([], ())),
],
'/jjb_templates': [
('/jjb_templates', (['dir1', 'dir2', 'dir3'], ())),
('/jjb_templates/dir1', (['bar'], ())),
('/jjb_templates/dir2', (['baz'], ())),
('/jjb_templates/dir3', ([], ())),
('/jjb_templates/dir1/bar', ([], ())),
('/jjb_templates/dir2/baz', ([], ())),
],
'/jjb_macros': [
('/jjb_macros', (['dir1', 'dir2', 'dir3'], ())),
('/jjb_macros/dir1', (['bar'], ())),
('/jjb_macros/dir2', (['baz'], ())),
('/jjb_macros/dir3', ([], ())),
('/jjb_macros/dir1/bar', ([], ())),
('/jjb_macros/dir2/baz', ([], ())),
],
}
def os_walk_side_effects(path_name, topdown):
return fake_os_walk(os_walk_return_values[path_name])(path_name, topdown)
@mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock)
class TestConfigs(CmdTestsBase):
def test_use_global_config(self):
"""
Verify that JJB uses the global config file by default
"""
args = self.parser.parse_args(['test', 'foo'])
self.assertEqual(cmd.get_config_file(args),
'/etc/jenkins_jobs/jenkins_jobs.ini')
def test_use_config_in_user_home(self):
"""
Verify that JJB uses config file in user home folder
"""
args = self.parser.parse_args(['test', 'foo'])
# args.output_dir = mock.MagicMock()
# mock_isfile.side_effect = True
expected_loc = os.path.join(os.path.expanduser('~'), '.config',
'jenkins_jobs', 'jenkins_jobs.ini')
with patch('os.path.isfile', return_value=True):
self.assertEqual(cmd.get_config_file(args), expected_loc)
@mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock)
class TestTests(CmdTestsBase):
def test_non_existing_config_dir(self):
"""
Run test mode and pass a non-existing configuration directory
"""
args = self.parser.parse_args(['test', 'foo'])
args.output_dir = mock.MagicMock()
self.assertRaises(IOError, cmd.execute, args, self.config)
def test_non_existing_config_file(self):
"""
Run test mode and pass a non-existing configuration file
"""
args = self.parser.parse_args(['test', 'non-existing.yaml'])
args.output_dir = mock.MagicMock()
self.assertRaises(IOError, cmd.execute, args, self.config)
def test_non_existing_job(self):
"""
Run test mode and pass a non-existing job name
(probably better to fail here)
"""
args = self.parser.parse_args(['test',
os.path.join(self.fixtures_path,
'cmd-001.yaml'),
'invalid'])
args.output_dir = mock.MagicMock(wraps=io.BytesIO())
cmd.execute(args, self.config) # probably better to fail here
args = ['--conf', self.default_config_file, 'test',
os.path.join(self.fixtures_path,
'cmd-001.yaml'),
'invalid']
self.execute_jenkins_jobs_with_args(args)
def test_valid_job(self):
"""
Run test mode and pass a valid job name
"""
args = self.parser.parse_args(['test',
os.path.join(self.fixtures_path,
'cmd-001.yaml'),
'foo-job'])
args.output_dir = mock.Mock(wraps=io.BytesIO())
cmd.execute(args, self.config) # probably better to fail here
@mock.patch('jenkins_jobs.cmd.Builder.update_jobs')
def test_multi_path(self, update_jobs_mock):
"""
Run test mode and pass multiple paths.
"""
path_list = list(os_walk_return_values.keys())
multipath = os.pathsep.join(path_list)
args = self.parser.parse_args(['test', multipath])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config)
self.assertEqual(args.path, path_list)
update_jobs_mock.assert_called_with(path_list, [],
output=args.output_dir,
n_workers=mock.ANY)
@mock.patch('jenkins_jobs.cmd.Builder.update_jobs')
@mock.patch('jenkins_jobs.cmd.os.path.isdir')
@mock.patch('jenkins_jobs.cmd.os.walk')
def test_recursive_multi_path(self, os_walk_mock, isdir_mock,
update_jobs_mock):
"""
Run test mode and pass multiple paths with recursive path option.
"""
os_walk_mock.side_effect = os_walk_side_effects
isdir_mock.return_value = True
path_list = os_walk_return_values.keys()
paths = []
for path in path_list:
paths.extend([p for p, _ in os_walk_return_values[path]])
multipath = os.pathsep.join(path_list)
args = self.parser.parse_args(['test', '-r', multipath])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config)
update_jobs_mock.assert_called_with(paths, [], output=args.output_dir,
n_workers=mock.ANY)
args = self.parser.parse_args(['test', multipath])
args.output_dir = mock.MagicMock()
self.config.set('job_builder', 'recursive', 'True')
cmd.execute(args, self.config)
update_jobs_mock.assert_called_with(paths, [], output=args.output_dir,
n_workers=mock.ANY)
@mock.patch('jenkins_jobs.cmd.Builder.update_jobs')
@mock.patch('jenkins_jobs.cmd.os.path.isdir')
@mock.patch('jenkins_jobs.cmd.os.walk')
def test_recursive_multi_path_with_excludes(self, os_walk_mock, isdir_mock,
update_jobs_mock):
"""
Run test mode and pass multiple paths with recursive path option.
"""
os_walk_mock.side_effect = os_walk_side_effects
isdir_mock.return_value = True
path_list = os_walk_return_values.keys()
paths = []
for path in path_list:
paths.extend([p for p, __ in os_walk_return_values[path]
if 'dir1' not in p and 'dir2' not in p])
multipath = os.pathsep.join(path_list)
args = self.parser.parse_args(['test', '-r', multipath, '-x',
'dir1:dir2'])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config)
update_jobs_mock.assert_called_with(paths, [], output=args.output_dir,
n_workers=mock.ANY)
args = ['--conf', self.default_config_file, 'test',
os.path.join(self.fixtures_path,
'cmd-001.yaml'),
'foo-job']
self.execute_jenkins_jobs_with_args(args)
def test_console_output(self):
"""
@ -197,8 +46,9 @@ class TestTests(CmdTestsBase):
console_out = io.BytesIO()
with mock.patch('sys.stdout', console_out):
cmd.main(['test', os.path.join(self.fixtures_path,
'cmd-001.yaml')])
args = ['--conf', self.default_config_file, 'test',
os.path.join(self.fixtures_path, 'cmd-001.yaml')]
self.execute_jenkins_jobs_with_args(args)
xml_content = io.open(os.path.join(self.fixtures_path, 'cmd-001.xml'),
'r', encoding='utf-8').read()
self.assertEqual(console_out.getvalue().decode('utf-8'), xml_content)
@ -214,7 +64,8 @@ class TestTests(CmdTestsBase):
with io.open(input_file, 'r') as f:
with mock.patch('sys.stdout', console_out):
with mock.patch('sys.stdin', f):
cmd.main(['test'])
args = ['--conf', self.default_config_file, 'test']
self.execute_jenkins_jobs_with_args(args)
xml_content = io.open(os.path.join(self.fixtures_path, 'cmd-001.xml'),
'r', encoding='utf-8').read()
@ -233,7 +84,8 @@ class TestTests(CmdTestsBase):
with io.open(input_file, 'r') as f:
with mock.patch('sys.stdout', console_out):
with mock.patch('sys.stdin', f):
cmd.main(['test'])
args = ['--conf', self.default_config_file, 'test']
self.execute_jenkins_jobs_with_args(args)
xml_content = io.open(os.path.join(self.fixtures_path, 'cmd-001.xml'),
'r', encoding='utf-8').read()
@ -253,24 +105,11 @@ class TestTests(CmdTestsBase):
with io.open(input_file, 'r', encoding='utf-8') as f:
with mock.patch('sys.stdout', console_out):
with mock.patch('sys.stdin', f):
e = self.assertRaises(UnicodeError, cmd.main, ['test'])
args = ['--conf', self.default_config_file, 'test']
jenkins_jobs = entry.JenkinsJobs(args)
e = self.assertRaises(UnicodeError, jenkins_jobs.execute)
self.assertIn("'ascii' codec can't encode character", str(e))
def test_config_with_test(self):
"""
Run test mode and pass a config file
"""
args = self.parser.parse_args(['--conf',
os.path.join(self.fixtures_path,
'cmd-001.conf'),
'test',
os.path.join(self.fixtures_path,
'cmd-001.yaml'),
'foo-job'])
config = cmd.setup_config_settings(args)
self.assertEqual(config.get('jenkins', 'url'),
"http://test-jenkins.with.non.default.url:8080/")
@mock.patch('jenkins_jobs.builder.YamlParser.generateXML')
@mock.patch('jenkins_jobs.parser.ModuleRegistry')
def test_plugins_info_stub_option(self, registry_mock, generateXML_mock):
@ -285,16 +124,15 @@ class TestTests(CmdTestsBase):
'-p',
plugins_info_stub_yaml_file,
os.path.join(self.fixtures_path, 'cmd-001.yaml')]
args = self.parser.parse_args(args)
with mock.patch('sys.stdout'):
cmd.execute(args, self.config) # probably better to fail here
self.execute_jenkins_jobs_with_args(args)
with io.open(plugins_info_stub_yaml_file,
'r', encoding='utf-8') as yaml_file:
plugins_info_list = yaml.load(yaml_file)
registry_mock.assert_called_with(self.config, plugins_info_list)
registry_mock.assert_called_with(mock.ANY,
plugins_info_list)
@mock.patch('jenkins_jobs.builder.YamlParser.generateXML')
@mock.patch('jenkins_jobs.parser.ModuleRegistry')
@ -312,11 +150,10 @@ class TestTests(CmdTestsBase):
'-p',
plugins_info_stub_yaml_file,
os.path.join(self.fixtures_path, 'cmd-001.yaml')]
args = self.parser.parse_args(args)
with mock.patch('sys.stdout'):
e = self.assertRaises(JenkinsJobsException, cmd.execute,
args, self.config)
jenkins_jobs = entry.JenkinsJobs(args)
e = self.assertRaises(JenkinsJobsException, jenkins_jobs.execute)
self.assertIn("must contain a Yaml list", str(e))
@ -341,8 +178,9 @@ class TestJenkinsGetPluginInfoError(CmdTestsBase):
jenkins.JenkinsException("Connection refused")
with mock.patch('sys.stdout'):
try:
cmd.main(['test', os.path.join(self.fixtures_path,
'cmd-001.yaml')])
args = ['--conf', self.default_config_file, 'test',
os.path.join(self.fixtures_path, 'cmd-001.yaml')]
self.execute_jenkins_jobs_with_args(args)
except jenkins.JenkinsException:
self.fail("jenkins.JenkinsException propagated to main")
except:
@ -356,8 +194,9 @@ class TestJenkinsGetPluginInfoError(CmdTestsBase):
plugins will be skipped when run if no config file provided.
"""
with mock.patch('sys.stdout', new_callable=io.BytesIO):
cmd.main(['test', os.path.join(self.fixtures_path,
'cmd-001.yaml')])
args = ['--conf', self.default_config_file, 'test',
os.path.join(self.fixtures_path, 'cmd-001.yaml')]
entry.JenkinsJobs(args)
self.assertFalse(get_plugins_info_mock.called)
@mock.patch('jenkins.Jenkins.get_plugins_info')
@ -368,9 +207,143 @@ class TestJenkinsGetPluginInfoError(CmdTestsBase):
querying through a config option.
"""
with mock.patch('sys.stdout', new_callable=io.BytesIO):
cmd.main(['--conf',
os.path.join(self.fixtures_path,
'disable-query-plugins.conf'),
'test',
os.path.join(self.fixtures_path, 'cmd-001.yaml')])
args = ['--conf',
os.path.join(self.fixtures_path,
'disable-query-plugins.conf'),
'test',
os.path.join(self.fixtures_path, 'cmd-001.yaml')]
entry.JenkinsJobs(args)
self.assertFalse(get_plugins_info_mock.called)
class MatchesDirMissingFilesMismatch(object):
def __init__(self, left_directory, right_directory):
self.left_directory = left_directory
self.right_directory = right_directory
def describe(self):
return "{0} and {1} contain different files".format(
self.left_directory,
self.right_directory)
def get_details(self):
return {}
class MatchesDirFileContentsMismatch(object):
def __init__(self, left_file, right_file):
self.left_file = left_file
self.right_file = right_file
def describe(self):
left_contents = open(self.left_file).readlines()
right_contents = open(self.right_file).readlines()
return "{0} is not equal to {1}:\n{2}".format(
difflib.unified_diff(left_contents, right_contents,
fromfile=self.left_file,
tofile=self.right_file),
self.left_file,
self.right_file)
def get_details(self):
return {}
class MatchesDir(object):
def __init__(self, directory):
self.__directory = directory
self.__files = self.__get_files(directory)
def __get_files(self, directory):
for root, _, files in os.walk(directory):
return files
def __str__(self,):
return "MatchesDir({0})".format(self.__dirname)
def match(self, other_directory):
other_files = self.__get_files(other_directory)
self.__files.sort()
other_files.sort()
if self.__files != other_files:
return MatchesDirMissingFilesMismatch(self.__directory,
other_directory)
for i, file in enumerate(self.__files):
my_file = os.path.join(self.__directory, file)
other_file = os.path.join(other_directory, other_files[i])
if not filecmp.cmp(my_file, other_file):
return MatchesDirFileContentsMismatch(my_file, other_file)
return None
@mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock)
class TestTestsMultiPath(CmdTestsBase):
def setUp(self):
super(TestTestsMultiPath, self).setUp()
path_list = [os.path.join(self.fixtures_path,
'multi-path/yamldirs/', p)
for p in ['dir1', 'dir2']]
self.multipath = os.pathsep.join(path_list)
self.output_dir = tempfile.mkdtemp()
def check_dirs_match(self, expected_dir):
try:
self.assertThat(self.output_dir, MatchesDir(expected_dir))
except testtools.matchers.MismatchError as e:
raise e
else:
shutil.rmtree(self.output_dir)
def test_multi_path(self):
"""
Run test mode and pass multiple paths.
"""
args = ['--conf', self.default_config_file, 'test',
'-o', self.output_dir, self.multipath]
self.execute_jenkins_jobs_with_args(args)
self.check_dirs_match(os.path.join(self.fixtures_path,
'multi-path/output_simple'))
def test_recursive_multi_path_command_line(self):
"""
Run test mode and pass multiple paths with recursive path option.
"""
args = ['--conf', self.default_config_file, 'test',
'-o', self.output_dir, '-r', self.multipath]
self.execute_jenkins_jobs_with_args(args)
self.check_dirs_match(os.path.join(self.fixtures_path,
'multi-path/output_recursive'))
def test_recursive_multi_path_config_file(self):
# test recursive set in configuration file
args = ['--conf', os.path.join(self.fixtures_path,
'multi-path/builder-recursive.ini'),
'test', '-o', self.output_dir, self.multipath]
self.execute_jenkins_jobs_with_args(args)
self.check_dirs_match(os.path.join(self.fixtures_path,
'multi-path/output_recursive'))
def test_recursive_multi_path_with_excludes(self):
"""
Run test mode and pass multiple paths with recursive path option.
"""
exclude_path = os.path.join(self.fixtures_path,
'multi-path/yamldirs/dir2/dir1')
args = ['--conf', self.default_config_file, 'test',
'-x', exclude_path,
'-o', self.output_dir,
'-r', self.multipath]
self.execute_jenkins_jobs_with_args(args)
self.check_dirs_match(
os.path.join(self.fixtures_path,
'multi-path/output_recursive_with_excludes'))

@ -18,7 +18,6 @@ import os
import six
from jenkins_jobs import builder
from jenkins_jobs import cmd
from tests.base import mock
from tests.cmd.test_cmd import CmdTestsBase
@ -35,9 +34,9 @@ class UpdateTests(CmdTestsBase):
update_jobs_mock.return_value = ([], 0)
path = os.path.join(self.fixtures_path, 'cmd-002.yaml')
args = self.parser.parse_args(['update', path])
args = ['--conf', self.default_config_file, 'update', path]
cmd.execute(args, self.config)
self.execute_jenkins_jobs_with_args(args)
update_jobs_mock.assert_called_with([path], [], n_workers=mock.ANY)
@mock.patch('jenkins_jobs.builder.Jenkins.is_job', return_value=True)
@ -54,9 +53,9 @@ class UpdateTests(CmdTestsBase):
update_job_mock.return_value = ([], 0)
path = os.path.join(self.fixtures_path, 'cmd-002.yaml')
args = self.parser.parse_args(['update', path])
args = ['--conf', self.default_config_file, 'update', path]
cmd.execute(args, self.config)
self.execute_jenkins_jobs_with_args(args)
self.assertTrue(isinstance(update_job_mock.call_args[0][1],
six.text_type))
@ -101,17 +100,18 @@ class UpdateTests(CmdTestsBase):
[True] * 2 + [False] * 2)
path = os.path.join(self.fixtures_path, 'cmd-002.yaml')
args = self.parser.parse_args(['update', '--delete-old', path])
args = ['--conf', self.default_config_file, 'update', '--delete-old',
path]
with mock.patch('jenkins_jobs.builder.Jenkins.update_job') as update:
with mock.patch('jenkins_jobs.builder.Jenkins.is_managed',
return_value=True):
cmd.execute(args, self.config)
self.assertEqual(2, update.call_count,
"Expected Jenkins.update_job to be called '%d' "
"times, got '%d' calls instead.\n"
"Called with: %s" % (2, update.call_count,
update.mock_calls))
self.execute_jenkins_jobs_with_args(args)
self.assertEquals(2, update.call_count,
"Expected Jenkins.update_job to be called '%d' "
"times, got '%d' calls instead.\n"
"Called with: %s" % (2, update.call_count,
update.mock_calls))
calls = [mock.call(name) for name in jobs]
self.assertEqual(2, delete_job_mock.call_count,
@ -130,11 +130,11 @@ class UpdateTests(CmdTestsBase):
"""
path = os.path.join(self.fixtures_path, 'cmd-002.yaml')
args = self.parser.parse_args(['update', path])
args = ['--conf', self.default_config_file, 'update', path]
with mock.patch('jenkins_jobs.cmd.Builder.update_job') as update_mock:
update_mock.return_value = ([], 0)
cmd.execute(args, self.config)
self.execute_jenkins_jobs_with_args(args)
# unless the timeout is set, should only call with 3 arguments
# (url, user, password)
self.assertEqual(len(jenkins_mock.call_args[0]), 3)
@ -148,12 +148,13 @@ class UpdateTests(CmdTestsBase):
"""
path = os.path.join(self.fixtures_path, 'cmd-002.yaml')
args = self.parser.parse_args(['update', path])
self.config.set('jenkins', 'timeout', '0.2')
config_file = os.path.join(self.fixtures_path,
'non-default-timeout.ini')
args = ['--conf', config_file, 'update', path]
with mock.patch('jenkins_jobs.cmd.Builder.update_job') as update_mock:
update_mock.return_value = ([], 0)
cmd.execute(args, self.config)
self.execute_jenkins_jobs_with_args(args)
# when timeout is set, the fourth argument to the Jenkins api init
# should be the value specified from the config
self.assertEqual(jenkins_mock.call_args[0][3], 0.2)

@ -1,10 +1,7 @@
import os
from six.moves import configparser
from six.moves import StringIO
import testtools
from jenkins_jobs import cmd
from jenkins_jobs.cli import entry
from tests.base import LoggingFixture