Add plugins_info to module registry object.

This makes it available for use by any module that has version-specific behavior
differences since modules always have access to the yamlparser which in turn
contains a module registry.

Change-Id: I1cae480a9a341ec2f6062904c962530dfce95057

jenkins_jobs/builder.py

@@ -131,12 +131,12 @@ def matches(what, glob_patterns):
 
 
 class YamlParser(object):
-    def __init__(self, config=None):
+    def __init__(self, config=None, plugins_info=None):
         self.data = {}
         self.jobs = []
         self.xml_jobs = []
         self.config = config
-        self.registry = ModuleRegistry(self.config)
+        self.registry = ModuleRegistry(self.config, plugins_info)
         self.path = ["."]
         if self.config:
             if config.has_section('job_builder') and \
@@ -406,12 +406,17 @@ class YamlParser(object):
 class ModuleRegistry(object):
     entry_points_cache = {}
 
-    def __init__(self, config):
+    def __init__(self, config, plugins_list=None):
         self.modules = []
         self.modules_by_component_type = {}
         self.handlers = {}
         self.global_config = config
 
+        if plugins_list is None:
+            self.plugins_dict = {}
+        else:
+            self.plugins_dict = self._get_plugins_info_dict(plugins_list)
+
         for entrypoint in pkg_resources.iter_entry_points(
                 group='jenkins_jobs.modules'):
             Mod = entrypoint.load()
@@ -421,6 +426,64 @@ class ModuleRegistry(object):
             if mod.component_type is not None:
                 self.modules_by_component_type[mod.component_type] = mod
 
+    @staticmethod
+    def _get_plugins_info_dict(plugins_list):
+        def mutate_plugin_info(plugin_info):
+            """
+            We perform mutations on a single member of plugin_info here, then
+            return a dictionary with the longName and shortName of the plugin
+            mapped to its plugin info dictionary.
+            """
+            version = plugin_info.get('version', '0')
+            plugin_info['version'] = re.sub(r'(.*)-(?:SNAPSHOT|BETA)',
+                                            r'\g<1>.preview', version)
+
+            aliases = []
+            for key in ['longName', 'shortName']:
+                value = plugin_info.get(key, None)
+                if value is not None:
+                    aliases.append(value)
+
+            plugin_info_dict = {}
+            for name in aliases:
+                plugin_info_dict[name] = plugin_info
+
+            return plugin_info_dict
+
+        list_of_dicts = [mutate_plugin_info(v) for v in plugins_list]
+
+        plugins_info_dict = {}
+        for d in list_of_dicts:
+            plugins_info_dict.update(d)
+
+        return plugins_info_dict
+
+    def get_plugin_info(self, plugin_name):
+        """ This method is intended to provide information about plugins within
+        a given module's implementation of Base.gen_xml. The return value is a
+        dictionary with data obtained directly from a running Jenkins instance.
+        This allows module authors to differentiate generated XML output based
+        on information such as specific plugin versions.
+
+        :arg string plugin_name: Either the shortName or longName of a plugin
+          as see in a query that looks like:
+          http://<jenkins-hostname>/pluginManager/api/json?pretty&depth=2
+
+        During a 'test' run, it is possible to override JJB's query to a live
+        Jenkins instance by passing it a path to a file containing a YAML list
+        of dictionaries that mimics the plugin properties you want your test
+        output to reflect::
+
+          jenkins-jobs test -p /path/to/plugins-info.yaml
+
+        Below is example YAML that might be included in
+        /path/to/plugins-info.yaml.
+
+        .. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml
+
+        """
+        return self.plugins_dict.get(plugin_name, {})
+
     def registerHandler(self, category, name, method):
         cat_dict = self.handlers.get(category, {})
         if not cat_dict:
@@ -612,6 +675,26 @@ class Jenkins(object):
             logger.info("Deleting jenkins job {0}".format(job_name))
             self.jenkins.delete_job(job_name)
 
+    def get_plugins_info(self):
+        """ Return a list of plugin_info dicts, one for each plugin on the
+        Jenkins instance.
+        """
+        try:
+            plugins_list = self.jenkins.get_plugins_info()
+        except jenkins.JenkinsException as e:
+            if re.search("Connection refused", str(e)):
+                logger.warn("Unable to retrieve Jenkins Plugin Info from {0},"
+                            " using default empty plugins info list.".format(
+                                self.jenkins.server))
+                plugins_list = [{'shortName': '',
+                                 'version': '',
+                                 'longName': ''}]
+            else:
+                raise e
+        logger.debug("Jenkins Plugin Info {0}".format(pformat(plugins_list)))
+
+        return plugins_list
+
     def get_jobs(self):
         return self.jenkins.get_jobs()
 
@@ -628,14 +711,20 @@ class Jenkins(object):
 
 class Builder(object):
     def __init__(self, jenkins_url, jenkins_user, jenkins_password,
-                 config=None, ignore_cache=False, flush_cache=False):
+                 config=None, ignore_cache=False, flush_cache=False,
+                 plugins_list=None):
         self.jenkins = Jenkins(jenkins_url, jenkins_user, jenkins_password)
         self.cache = CacheStorage(jenkins_url, flush=flush_cache)
         self.global_config = config
         self.ignore_cache = ignore_cache
 
+        if plugins_list is None:
+            self.plugins_list = self.jenkins.get_plugins_info()
+        else:
+            self.plugins_list = plugins_list
+
     def load_files(self, fn):
-        self.parser = YamlParser(self.global_config)
+        self.parser = YamlParser(self.global_config, self.plugins_list)
 
         # handle deprecated behavior
         if not hasattr(fn, '__iter__'):

jenkins_jobs/cmd.py

@@ -19,6 +19,7 @@ import logging
 import os
 import platform
 import sys
+import yaml
 import jenkins_jobs.version
 
 from jenkins_jobs.builder import Builder
@@ -69,6 +70,9 @@ def create_parser():
                                   help='look for yaml files recursively')
     subparser = parser.add_subparsers(help='update, test or delete job',
                                       dest='command')
+
+    # subparser: update
+
     parser_update = subparser.add_parser('update', parents=[recursive_parser])
     parser_update.add_argument('path', help='colon-separated list of paths to'
                                             ' YAML files or directories')
@@ -76,18 +80,29 @@ def create_parser():
     parser_update.add_argument('--delete-old', help='delete obsolete jobs',
                                action='store_true',
                                dest='delete_old', default=False,)
+
+    # subparser: test
+
     parser_test = subparser.add_parser('test', parents=[recursive_parser])
     parser_test.add_argument('path', help='colon-separated list of paths to'
                                           ' YAML files or directories',
                              nargs='?', default=sys.stdin)
+    parser_test.add_argument('-p', dest='plugins_info_path', default=None,
+                             help='path to plugin info YAML file')
     parser_test.add_argument('-o', dest='output_dir', default=sys.stdout,
                              help='path to output XML')
     parser_test.add_argument('name', help='name(s) of job(s)', nargs='*')
+
+    # subparser: delete
+
     parser_delete = subparser.add_parser('delete')
     parser_delete.add_argument('name', help='name of job', nargs='+')
     parser_delete.add_argument('-p', '--path', default=None,
                                help='colon-separated list of paths to'
                                     ' YAML files or directories')
+
+    # subparser: delete-all
+
     subparser.add_parser('delete-all',
                          help='delete *ALL* jobs from Jenkins server, '
                          'including those not managed by Jenkins Job '
@@ -186,12 +201,22 @@ def execute(options, config):
     except (TypeError, configparser.NoOptionError):
         password = None
 
+    plugins_info = None
+
+    if getattr(options, 'plugins_info_path', None) is not None:
+        with open(options.plugins_info_path, 'r') as yaml_file:
+            plugins_info = yaml.load(yaml_file)
+        if not isinstance(plugins_info, list):
+            raise JenkinsJobsException("{0} must contain a Yaml list!"
+                                       .format(options.plugins_info_path))
+
     builder = Builder(config.get('jenkins', 'url'),
                       user,
                       password,
                       config,
                       ignore_cache=ignore_cache,
-                      flush_cache=options.flush_cache)
+                      flush_cache=options.flush_cache,
+                      plugins_list=plugins_info)
 
     if getattr(options, 'path', None):
         if options.path == sys.stdin:

tests/base.py

@@ -25,6 +25,7 @@ import doctest
 import json
 import operator
 import testtools
+from testtools.content import text_content
 import xml.etree.ElementTree as XML
 from six.moves import configparser
 # This dance deals with the fact that we want unittest.mock if
@@ -41,7 +42,8 @@ from jenkins_jobs.modules import (project_flow,
                                   project_multijob)
 
 
-def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml'):
+def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml',
+                  plugins_info_ext='plugins_info.yaml'):
     """Returns a list of scenarios, each scenario being described
     by two parameters (yaml and xml filenames by default).
         - content of the fixture output file (aka expected)
@@ -51,6 +53,9 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml'):
     input_files = [f for f in files if re.match(r'.*\.{0}$'.format(in_ext), f)]
 
     for input_filename in input_files:
+        if input_filename.endswith(plugins_info_ext):
+            continue
+
         output_candidate = re.sub(r'\.{0}$'.format(in_ext),
                                   '.{0}'.format(out_ext), input_filename)
         # Make sure the input file has a output counterpart
@@ -60,6 +65,12 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml'):
                 .format(out_ext.upper(), output_candidate,
                         in_ext.upper(), input_filename))
 
+        plugins_info_candidate = re.sub(r'\.{0}$'.format(in_ext),
+                                        '.{0}'.format(plugins_info_ext),
+                                        input_filename)
+        if plugins_info_candidate not in files:
+            plugins_info_candidate = None
+
         conf_candidate = re.sub(r'\.yaml$', '.conf', input_filename)
         # If present, add the configuration file
         if conf_candidate not in files:
@@ -69,6 +80,7 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml'):
             'in_filename': input_filename,
             'out_filename': output_candidate,
             'conf_filename': conf_candidate,
+            'plugins_info_filename': plugins_info_candidate,
         }))
 
     return scenarios
@@ -90,8 +102,8 @@ class BaseTestCase(object):
         xml_content = u"%s" % codecs.open(xml_filepath, 'r', 'utf-8').read()
         return xml_content
 
-    def _read_yaml_content(self):
-        yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
+    def _read_yaml_content(self, filename):
+        yaml_filepath = os.path.join(self.fixtures_path, filename)
         with open(yaml_filepath, 'r') as yaml_file:
             yaml_content = yaml.load(yaml_file)
         return yaml_content
@@ -100,8 +112,16 @@ class BaseTestCase(object):
         if not self.out_filename or not self.in_filename:
             return
 
+        if self.conf_filename is not None:
+            config = configparser.ConfigParser()
+            conf_filepath = os.path.join(self.fixtures_path,
+                                         self.conf_filename)
+            config.readfp(open(conf_filepath))
+        else:
+            config = {}
+
         expected_xml = self._read_utf8_content()
-        yaml_content = self._read_yaml_content()
+        yaml_content = self._read_yaml_content(self.in_filename)
         project = None
         if ('project-type' in yaml_content):
             if (yaml_content['project-type'] == "maven"):
@@ -118,7 +138,16 @@ class BaseTestCase(object):
         else:
             xml_project = XML.Element('project')
         parser = YamlParser()
-        pub = self.klass(ModuleRegistry({}))
+
+        plugins_info = None
+        if self.plugins_info_filename is not None:
+            plugins_info = self._read_yaml_content(self.plugins_info_filename)
+            self.addDetail("plugins-info-filename",
+                           text_content(self.plugins_info_filename))
+            self.addDetail("plugins-info",
+                           text_content(str(plugins_info)))
+
+        pub = self.klass(ModuleRegistry(config, plugins_info))
 
         # Generate the XML tree directly with modules/general
         pub.gen_xml(parser, xml_project, yaml_content)
@@ -174,7 +203,7 @@ class JsonTestCase(BaseTestCase):
 
     def test_yaml_snippet(self):
         expected_json = self._read_utf8_content()
-        yaml_content = self._read_yaml_content()
+        yaml_content = self._read_yaml_content(self.in_filename)
 
         pretty_json = json.dumps(yaml_content, indent=4,
                                  separators=(',', ': '))

tests/cmd/fixtures/bogus-plugins-info.yaml

@@ -0,0 +1,3 @@
+longName: 'Jenkins HipChat Plugin'
+shortName: 'hipchat'
+version: "0.1.8"

tests/cmd/fixtures/plugins-info.yaml

@@ -0,0 +1,3 @@
+- longName: 'Jenkins HipChat Plugin'
+  shortName: 'hipchat'
+  version: "0.1.8"

tests/cmd/subcommands/test_delete.py

@@ -4,6 +4,7 @@ from tests.base import mock
 from tests.cmd.test_cmd import CmdTestsBase
 
 
+@mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock)
 class DeleteTests(CmdTestsBase):
 
     @mock.patch('jenkins_jobs.cmd.Builder.delete_job')

tests/cmd/subcommands/test_test.py

@@ -1,9 +1,14 @@
 import os
 import io
 import codecs
+import yaml
+
+import jenkins
+
 from jenkins_jobs import cmd
-from tests.base import mock
+from jenkins_jobs.errors import JenkinsJobsException
 from tests.cmd.test_cmd import CmdTestsBase
+from tests.base import mock
 
 
 os_walk_return_values = {
@@ -38,6 +43,7 @@ def os_walk_side_effects(path_name, topdown):
     return os_walk_return_values[path_name]
 
 
+@mock.patch('jenkins_jobs.builder.Jenkins.get_plugins_info', mock.MagicMock)
 class TestTests(CmdTestsBase):
 
     def test_non_existing_config_dir(self):
@@ -154,3 +160,79 @@ class TestTests(CmdTestsBase):
         config = cmd.setup_config_settings(args)
         self.assertEqual(config.get('jenkins', 'url'),
                          "http://test-jenkins.with.non.default.url:8080/")
+
+    @mock.patch('jenkins_jobs.builder.YamlParser.generateXML')
+    @mock.patch('jenkins_jobs.builder.ModuleRegistry')
+    def test_plugins_info_stub_option(self, registry_mock, generateXML_mock):
+        """
+        Test handling of plugins_info stub option.
+        """
+        plugins_info_stub_yaml_file = os.path.join(self.fixtures_path,
+                                                   'plugins-info.yaml')
+        args = ['--conf',
+                os.path.join(self.fixtures_path, 'cmd-001.conf'),
+                'test',
+                '-p',
+                plugins_info_stub_yaml_file,
+                os.path.join(self.fixtures_path, 'cmd-001.yaml')]
+        args = self.parser.parse_args(args)
+
+        with mock.patch('sys.stdout'):
+            cmd.execute(args, self.config)   # probably better to fail here
+
+        with open(plugins_info_stub_yaml_file, 'r') as yaml_file:
+            plugins_info_list = yaml.load(yaml_file)
+
+        registry_mock.assert_called_with(self.config, plugins_info_list)
+
+    @mock.patch('jenkins_jobs.builder.YamlParser.generateXML')
+    @mock.patch('jenkins_jobs.builder.ModuleRegistry')
+    def test_bogus_plugins_info_stub_option(self, registry_mock,
+                                            generateXML_mock):
+        """
+        Verify that a JenkinsJobException is raised if the plugins_info stub
+        file does not yield a list as its top-level object.
+        """
+        plugins_info_stub_yaml_file = os.path.join(self.fixtures_path,
+                                                   'bogus-plugins-info.yaml')
+        args = ['--conf',
+                os.path.join(self.fixtures_path, 'cmd-001.conf'),
+                'test',
+                '-p',
+                plugins_info_stub_yaml_file,
+                os.path.join(self.fixtures_path, 'cmd-001.yaml')]
+        args = self.parser.parse_args(args)
+
+        with mock.patch('sys.stdout'):
+            e = self.assertRaises(JenkinsJobsException, cmd.execute,
+                                  args, self.config)
+        self.assertIn("must contain a Yaml list", str(e))
+
+
+class TestJenkinsGetPluginInfoError(CmdTestsBase):
+    """ This test class is used for testing the 'test' subcommand when we want
+    to validate its behavior without mocking
+    jenkins_jobs.builder.Jenkins.get_plugins_info
+    """
+
+    @mock.patch('jenkins.Jenkins.get_plugins_info')
+    def test_console_output_jenkins_connection_failure_warning(
+            self, get_plugins_info_mock):
+        """
+        Run test mode and verify that failed Jenkins connection attempt
+        exception does not bubble out of cmd.main. Ideally, we would also test
+        that an appropriate message is logged to stderr but it's somewhat
+        difficult to figure out how to actually enable stderr in this test
+        suite.
+        """
+
+        get_plugins_info_mock.side_effect = \
+            jenkins.JenkinsException("Connection refused")
+        with mock.patch('sys.stdout'):
+            try:
+                cmd.main(['test', os.path.join(self.fixtures_path,
+                                               'cmd-001.yaml')])
+            except jenkins.JenkinsException:
+                self.fail("jenkins.JenkinsException propagated to main")
+            except:
+                pass  # only care about jenkins.JenkinsException for now

tests/moduleregistry/test_moduleregistry.py

@@ -0,0 +1,119 @@
+import testtools as tt
+import pkg_resources
+from testtools.content import text_content
+from testscenarios.testcase import TestWithScenarios
+from six.moves import configparser, StringIO
+
+from jenkins_jobs import cmd
+from jenkins_jobs.builder import ModuleRegistry
+
+
+class ModuleRegistryPluginInfoTestsWithScenarios(TestWithScenarios,
+                                                 tt.TestCase):
+    scenarios = [
+        ('s1', dict(v1='1.0.0', op='__gt__', v2='0.8.0')),
+        ('s2', dict(v1='1.0.1alpha', op='__gt__', v2='1.0.0')),
+        ('s3', dict(v1='1.0', op='__eq__', v2='1.0.0')),
+        ('s4', dict(v1='1.0', op='__eq__', v2='1.0')),
+        ('s5', dict(v1='1.0', op='__lt__', v2='1.8.0')),
+        ('s6', dict(v1='1.0.1alpha', op='__lt__', v2='1.0.1')),
+        ('s7', dict(v1='1.0alpha', op='__lt__', v2='1.0.0')),
+        ('s8', dict(v1='1.0-alpha', op='__lt__', v2='1.0.0')),
+        ('s9', dict(v1='1.1-alpha', op='__gt__', v2='1.0')),
+        ('s10', dict(v1='1.0-SNAPSHOT', op='__lt__', v2='1.0')),
+        ('s11', dict(v1='1.0.preview', op='__lt__', v2='1.0')),
+        ('s12', dict(v1='1.1-SNAPSHOT', op='__gt__', v2='1.0')),
+        ('s13', dict(v1='1.0a-SNAPSHOT', op='__lt__', v2='1.0a')),
+    ]
+
+    def setUp(self):
+        super(ModuleRegistryPluginInfoTestsWithScenarios, self).setUp()
+
+        config = configparser.ConfigParser()
+        config.readfp(StringIO(cmd.DEFAULT_CONF))
+
+        plugin_info = [{'shortName': "HerpDerpPlugin",
+                        'longName': "Blah Blah Blah Plugin"
+                        }]
+        plugin_info.append({'shortName': "JankyPlugin1",
+                            'longName': "Not A Real Plugin",
+                            'version': self.v1
+                            })
+
+        self.addDetail("plugin_info", text_content(str(plugin_info)))
+        self.registry = ModuleRegistry(config, plugin_info)
+
+    def tearDown(self):
+        super(ModuleRegistryPluginInfoTestsWithScenarios, self).tearDown()
+
+    def test_get_plugin_info_dict(self):
+        """
+        The goal of this test is to validate that the plugin_info returned by
+        ModuleRegistry.get_plugin_info is a dictionary whose key 'shortName' is
+        the same value as the string argument passed to
+        ModuleRegistry.get_plugin_info.
+        """
+        plugin_name = "JankyPlugin1"
+        plugin_info = self.registry.get_plugin_info(plugin_name)
+
+        self.assertIsInstance(plugin_info, dict)
+        self.assertEqual(plugin_info['shortName'], plugin_name)
+
+    def test_get_plugin_info_dict_using_longName(self):
+        """
+        The goal of this test is to validate that the plugin_info returned by
+        ModuleRegistry.get_plugin_info is a dictionary whose key 'longName' is
+        the same value as the string argument passed to
+        ModuleRegistry.get_plugin_info.
+        """
+        plugin_name = "Blah Blah Blah Plugin"
+        plugin_info = self.registry.get_plugin_info(plugin_name)
+
+        self.assertIsInstance(plugin_info, dict)
+        self.assertEqual(plugin_info['longName'], plugin_name)
+
+    def test_get_plugin_info_dict_no_plugin(self):
+        """
+        The goal of this test case is to validate the behavior of
+        ModuleRegistry.get_plugin_info when the given plugin cannot be found in
+        ModuleRegistry's internal representation of the plugins_info.
+        """
+        plugin_name = "PluginDoesNotExist"
+        plugin_info = self.registry.get_plugin_info(plugin_name)
+
+        self.assertIsInstance(plugin_info, dict)
+        self.assertEqual(plugin_info, {})
+
+    def test_get_plugin_info_dict_no_version(self):
+        """
+        The goal of this test case is to validate the behavior of
+        ModuleRegistry.get_plugin_info when the given plugin shortName returns
+        plugin_info dict that has no version string. In a sane world where
+        plugin frameworks like Jenkins' are sane this should never happen, but
+        I am including this test and the corresponding default behavior
+        because, well, it's Jenkins.
+        """
+        plugin_name = "HerpDerpPlugin"
+        plugin_info = self.registry.get_plugin_info(plugin_name)
+
+        self.assertIsInstance(plugin_info, dict)
+        self.assertEqual(plugin_info['shortName'], plugin_name)
+        self.assertEqual(plugin_info['version'], '0')
+
+    def test_plugin_version_comparison(self):
+        """
+        The goal of this test case is to validate that valid tuple versions are
+        ordinally correct. That is, for each given scenario, v1.op(v2)==True
+        where 'op' is the equality operator defined for the scenario.
+        """
+        plugin_name = "JankyPlugin1"
+        plugin_info = self.registry.get_plugin_info(plugin_name)
+        v1 = plugin_info.get("version")
+
+        op = getattr(pkg_resources.parse_version(v1), self.op)
+        test = op(pkg_resources.parse_version(self.v2))
+
+        self.assertTrue(test,
+                        msg="Unexpectedly found {0} {2} {1} == False "
+                            "when comparing versions!"
+                            .format(v1, self.v2, self.op))