Tools to make Jenkins jobs from templates
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

base.py 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338
  1. #!/usr/bin/env python
  2. #
  3. # Joint copyright:
  4. # - Copyright 2012,2013 Wikimedia Foundation
  5. # - Copyright 2012,2013 Antoine "hashar" Musso
  6. # - Copyright 2013 Arnaud Fabre
  7. #
  8. # Licensed under the Apache License, Version 2.0 (the "License");
  9. # you may not use this file except in compliance with the License.
  10. # You may obtain a copy of the License at
  11. #
  12. # http://www.apache.org/licenses/LICENSE-2.0
  13. #
  14. # Unless required by applicable law or agreed to in writing, software
  15. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17. # License for the specific language governing permissions and limitations
  18. # under the License.
  19. import doctest
  20. import io
  21. import json
  22. import logging
  23. import os
  24. import re
  25. import xml.etree.ElementTree as XML
  26. import fixtures
  27. import six
  28. from six.moves import StringIO
  29. import testtools
  30. from testtools.content import text_content
  31. import testscenarios
  32. from yaml import safe_dump
  33. from jenkins_jobs.config import JJBConfig
  34. from jenkins_jobs.errors import InvalidAttributeError
  35. import jenkins_jobs.local_yaml as yaml
  36. from jenkins_jobs.alphanum import AlphanumSort
  37. from jenkins_jobs.modules import project_externaljob
  38. from jenkins_jobs.modules import project_flow
  39. from jenkins_jobs.modules import project_matrix
  40. from jenkins_jobs.modules import project_maven
  41. from jenkins_jobs.modules import project_multibranch
  42. from jenkins_jobs.modules import project_multijob
  43. from jenkins_jobs.modules import view_list
  44. from jenkins_jobs.modules import view_pipeline
  45. from jenkins_jobs.parser import YamlParser
  46. from jenkins_jobs.registry import ModuleRegistry
  47. from jenkins_jobs.xml_config import XmlJob
  48. from jenkins_jobs.xml_config import XmlJobGenerator
  49. # This dance deals with the fact that we want unittest.mock if
  50. # we're on Python 3.4 and later, and non-stdlib mock otherwise.
  51. try:
  52. from unittest import mock # noqa
  53. except ImportError:
  54. import mock # noqa
  55. def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml',
  56. plugins_info_ext='plugins_info.yaml',
  57. filter_func=None):
  58. """Returns a list of scenarios, each scenario being described
  59. by two parameters (yaml and xml filenames by default).
  60. - content of the fixture output file (aka expected)
  61. """
  62. scenarios = []
  63. files = {}
  64. for dirpath, _, fs in os.walk(fixtures_path):
  65. for fn in fs:
  66. if fn in files:
  67. files[fn].append(os.path.join(dirpath, fn))
  68. else:
  69. files[fn] = [os.path.join(dirpath, fn)]
  70. input_files = [files[f][0] for f in files if
  71. re.match(r'.*\.{0}$'.format(in_ext), f)]
  72. for input_filename in input_files:
  73. if input_filename.endswith(plugins_info_ext):
  74. continue
  75. if callable(filter_func) and filter_func(input_filename):
  76. continue
  77. output_candidate = re.sub(r'\.{0}$'.format(in_ext),
  78. '.{0}'.format(out_ext), input_filename)
  79. # assume empty file if no output candidate found
  80. if os.path.basename(output_candidate) in files:
  81. out_filenames = files[os.path.basename(output_candidate)]
  82. else:
  83. out_filenames = None
  84. plugins_info_candidate = re.sub(r'\.{0}$'.format(in_ext),
  85. '.{0}'.format(plugins_info_ext),
  86. input_filename)
  87. if os.path.basename(plugins_info_candidate) not in files:
  88. plugins_info_candidate = None
  89. conf_candidate = re.sub(r'\.yaml$|\.json$', '.conf', input_filename)
  90. conf_filename = files.get(os.path.basename(conf_candidate), None)
  91. if conf_filename:
  92. conf_filename = conf_filename[0]
  93. else:
  94. # for testing purposes we want to avoid using user config files
  95. conf_filename = os.devnull
  96. scenarios.append((input_filename, {
  97. 'in_filename': input_filename,
  98. 'out_filenames': out_filenames,
  99. 'conf_filename': conf_filename,
  100. 'plugins_info_filename': plugins_info_candidate,
  101. }))
  102. return scenarios
  103. class BaseTestCase(testtools.TestCase):
  104. # TestCase settings:
  105. maxDiff = None # always dump text difference
  106. longMessage = True # keep normal error message when providing our
  107. def setUp(self):
  108. super(BaseTestCase, self).setUp()
  109. self.logger = self.useFixture(fixtures.FakeLogger(level=logging.DEBUG))
  110. def _read_utf8_content(self):
  111. # if None assume empty file
  112. if not self.out_filenames:
  113. return u""
  114. # Read XML content, assuming it is unicode encoded
  115. xml_content = ""
  116. for f in sorted(self.out_filenames):
  117. xml_content += u"%s" % io.open(f, 'r', encoding='utf-8').read()
  118. return xml_content
  119. def _read_yaml_content(self, filename):
  120. with io.open(filename, 'r', encoding='utf-8') as yaml_file:
  121. yaml_content = yaml.load(yaml_file)
  122. return yaml_content
  123. def _get_config(self):
  124. jjb_config = JJBConfig(self.conf_filename)
  125. jjb_config.validate()
  126. return jjb_config
  127. class BaseScenariosTestCase(testscenarios.TestWithScenarios, BaseTestCase):
  128. scenarios = []
  129. fixtures_path = None
  130. def test_yaml_snippet(self):
  131. if not self.in_filename:
  132. return
  133. jjb_config = self._get_config()
  134. expected_xml = self._read_utf8_content()
  135. yaml_content = self._read_yaml_content(self.in_filename)
  136. plugins_info = None
  137. if self.plugins_info_filename:
  138. plugins_info = self._read_yaml_content(self.plugins_info_filename)
  139. self.addDetail("plugins-info-filename",
  140. text_content(self.plugins_info_filename))
  141. self.addDetail("plugins-info",
  142. text_content(str(plugins_info)))
  143. parser = YamlParser(jjb_config)
  144. registry = ModuleRegistry(jjb_config, plugins_info)
  145. registry.set_parser_data(parser.data)
  146. pub = self.klass(registry)
  147. project = None
  148. if ('project-type' in yaml_content):
  149. if (yaml_content['project-type'] == "maven"):
  150. project = project_maven.Maven(registry)
  151. elif (yaml_content['project-type'] == "matrix"):
  152. project = project_matrix.Matrix(registry)
  153. elif (yaml_content['project-type'] == "flow"):
  154. project = project_flow.Flow(registry)
  155. elif (yaml_content['project-type'] == "multijob"):
  156. project = project_multijob.MultiJob(registry)
  157. elif (yaml_content['project-type'] == "multibranch"):
  158. project = project_multibranch.WorkflowMultiBranch(registry)
  159. elif (yaml_content['project-type'] == "multibranch-defaults"):
  160. project = project_multibranch.WorkflowMultiBranchDefaults(registry) # noqa
  161. elif (yaml_content['project-type'] == "externaljob"):
  162. project = project_externaljob.ExternalJob(registry)
  163. if 'view-type' in yaml_content:
  164. if yaml_content['view-type'] == "list":
  165. project = view_list.List(None)
  166. elif yaml_content['view-type'] == "pipeline":
  167. project = view_pipeline.Pipeline(None)
  168. else:
  169. raise InvalidAttributeError(
  170. 'view-type', yaml_content['view-type'])
  171. if project:
  172. xml_project = project.root_xml(yaml_content)
  173. else:
  174. xml_project = XML.Element('project')
  175. # Generate the XML tree directly with modules/general
  176. pub.gen_xml(xml_project, yaml_content)
  177. # check output file is under correct path
  178. if 'name' in yaml_content:
  179. prefix = os.path.dirname(self.in_filename)
  180. # split using '/' since fullname uses URL path separator
  181. expected_folders = [os.path.normpath(
  182. os.path.join(prefix,
  183. '/'.join(parser._getfullname(yaml_content).
  184. split('/')[:-1])))]
  185. actual_folders = [os.path.dirname(f) for f in self.out_filenames]
  186. self.assertEquals(
  187. expected_folders, actual_folders,
  188. "Output file under wrong path, was '%s', should be '%s'" %
  189. (self.out_filenames[0],
  190. os.path.join(expected_folders[0],
  191. os.path.basename(self.out_filenames[0]))))
  192. # Prettify generated XML
  193. pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8')
  194. self.assertThat(
  195. pretty_xml,
  196. testtools.matchers.DocTestMatches(expected_xml,
  197. doctest.ELLIPSIS |
  198. doctest.REPORT_NDIFF)
  199. )
  200. class SingleJobTestCase(BaseScenariosTestCase):
  201. def test_yaml_snippet(self):
  202. config = self._get_config()
  203. expected_xml = self._read_utf8_content().strip()
  204. parser = YamlParser(config)
  205. parser.parse(self.in_filename)
  206. plugins_info = None
  207. if self.plugins_info_filename:
  208. plugins_info = self._read_yaml_content(self.plugins_info_filename)
  209. self.addDetail("plugins-info-filename",
  210. text_content(self.plugins_info_filename))
  211. self.addDetail("plugins-info",
  212. text_content(str(plugins_info)))
  213. registry = ModuleRegistry(config, plugins_info)
  214. registry.set_parser_data(parser.data)
  215. job_data_list, view_data_list = parser.expandYaml(registry)
  216. # Generate the XML tree
  217. xml_generator = XmlJobGenerator(registry)
  218. xml_jobs = xml_generator.generateXML(job_data_list)
  219. xml_jobs.sort(key=AlphanumSort)
  220. # check reference files are under correct path for folders
  221. prefix = os.path.dirname(self.in_filename)
  222. # split using '/' since fullname uses URL path separator
  223. expected_folders = list(set([
  224. os.path.normpath(
  225. os.path.join(prefix,
  226. '/'.join(job_data['name'].split('/')[:-1])))
  227. for job_data in job_data_list
  228. ]))
  229. actual_folders = [os.path.dirname(f) for f in self.out_filenames]
  230. six.assertCountEqual(
  231. self,
  232. expected_folders, actual_folders,
  233. "Output file under wrong path, was '%s', should be '%s'" %
  234. (self.out_filenames[0],
  235. os.path.join(expected_folders[0],
  236. os.path.basename(self.out_filenames[0]))))
  237. # Prettify generated XML
  238. pretty_xml = u"\n".join(job.output().decode('utf-8')
  239. for job in xml_jobs).strip()
  240. self.assertThat(
  241. pretty_xml,
  242. testtools.matchers.DocTestMatches(expected_xml,
  243. doctest.ELLIPSIS |
  244. doctest.REPORT_NDIFF)
  245. )
  246. class JsonTestCase(BaseScenariosTestCase):
  247. def test_yaml_snippet(self):
  248. expected_json = self._read_utf8_content()
  249. yaml_content = self._read_yaml_content(self.in_filename)
  250. pretty_json = json.dumps(yaml_content, indent=4,
  251. separators=(',', ': '))
  252. self.assertThat(
  253. pretty_json,
  254. testtools.matchers.DocTestMatches(expected_json,
  255. doctest.ELLIPSIS |
  256. doctest.REPORT_NDIFF)
  257. )
  258. class YamlTestCase(BaseScenariosTestCase):
  259. def test_yaml_snippet(self):
  260. expected_yaml = self._read_utf8_content()
  261. yaml_content = self._read_yaml_content(self.in_filename)
  262. # using json forces expansion of yaml anchors and aliases in the
  263. # outputted yaml, otherwise it would simply appear exactly as
  264. # entered which doesn't show that the net effect of the yaml
  265. data = StringIO(json.dumps(yaml_content))
  266. pretty_yaml = safe_dump(json.load(data), default_flow_style=False)
  267. self.assertThat(
  268. pretty_yaml,
  269. testtools.matchers.DocTestMatches(expected_yaml,
  270. doctest.ELLIPSIS |
  271. doctest.REPORT_NDIFF)
  272. )