Configuration files for project CI systems
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

320 lines
12 KiB

  1. #!/usr/bin/env python
  2. #
  3. # Check that gerrit/projects.yaml contains valid entries.
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import argparse
  17. import contextlib
  18. import git
  19. import os
  20. import re
  21. import shutil
  22. import sys
  23. import tempfile
  24. import yaml
  25. @contextlib.contextmanager
  26. def tempdir():
  27. try:
  28. reqroot = tempfile.mkdtemp()
  29. yield reqroot
  30. finally:
  31. shutil.rmtree(reqroot, ignore_errors=True)
  32. def check_repo(repo_path):
  33. found_errors = 0
  34. print("Checking git repo '%s':" % repo_path)
  35. with tempdir() as repopath:
  36. repo = git.Repo.clone_from(repo_path, repopath)
  37. remotes = repo.git.branch('--remote')
  38. branches = [r.strip() for r in remotes.splitlines() if r.strip()]
  39. print(" Remote branches:")
  40. for r in branches:
  41. print(" %s" % r)
  42. if 'origin/master' in branches:
  43. print(" Master branch exists.")
  44. else:
  45. found_errors += 1
  46. print(" ERROR: No master branch exists")
  47. if 'origin/stable' in branches:
  48. found_errors += 1
  49. print(" ERROR: A branch named 'stable' exists, this will"
  50. " break future\n"
  51. " creation of stable/RELEASEbranches.\n"
  52. " Delete the branch on your upstream project.")
  53. if 'origin/feature' in branches:
  54. found_errors += 1
  55. print(" ERROR: A branch named 'feature' exists, this will break "
  56. "future\n"
  57. " creation of feature/NAME branches.\n"
  58. " Delete the branch on your upstream project.")
  59. if repo.tags:
  60. print(" Found the following tags:")
  61. for tag in repo.tags:
  62. print(" %s" % tag)
  63. else:
  64. print(" Found no tags.")
  65. # Check that no zuul files are in here
  66. for branch in branches:
  67. print("Testing branch %s" % branch)
  68. if 'origin/HEAD' in branch:
  69. continue
  70. repo.git.checkout(branch)
  71. head = repo.head.commit.tree
  72. for z in ['zuul.yaml', '.zuul.yaml', 'zuul.d', '.zuul.d']:
  73. if z in head:
  74. found_errors += 1
  75. print(" ERROR: Found %s on branch %s" % (z, branch))
  76. print(" Remove any zuul config files before import.")
  77. # Just an empty line for nicer formatting
  78. print("")
  79. return found_errors
  80. # Check that name exists in set project_names
  81. def check_project_exists(name, project_names):
  82. if name not in project_names:
  83. print(" Error: project %s does not exist in gerrit" % name)
  84. return 1
  85. return 0
  86. def check_zuul_main(zuul_main, projects):
  87. found_errors = 0
  88. main_content = yaml.safe_load(open(zuul_main, 'r'))
  89. print("Checking %s" % zuul_main)
  90. project_names = set()
  91. for p in projects:
  92. name = p.get('project')
  93. project_names.add(name)
  94. # Check that for each gerrit source, we have a project defined in gerrit.
  95. for tenant in main_content:
  96. t = tenant.get('tenant')
  97. sources = t.get('source')
  98. if sources and sources.get('gerrit'):
  99. for project_types in sources['gerrit']:
  100. for entry in sources['gerrit'][project_types]:
  101. if isinstance(entry, dict):
  102. if 'projects' in entry:
  103. for x in entry['projects']:
  104. found_errors += check_project_exists(
  105. x, project_names)
  106. else:
  107. for x in entry.keys():
  108. found_errors += check_project_exists(
  109. x, project_names)
  110. else:
  111. found_errors += check_project_exists(
  112. entry, project_names)
  113. # Just an empty line for nicer formatting
  114. print("")
  115. return found_errors
  116. def main():
  117. found_errors = 0
  118. parser = argparse.ArgumentParser()
  119. parser.add_argument('-v', '--verbose',
  120. dest='verbose',
  121. default=False,
  122. action='store_true')
  123. parser.add_argument(
  124. 'infile',
  125. help='Path to gerrit/projects.yaml',
  126. )
  127. parser.add_argument(
  128. 'acldir',
  129. help='Path to gerrit/acl',
  130. )
  131. parser.add_argument(
  132. 'zuul_main_file',
  133. help='Path to zuul/main.yaml',
  134. )
  135. args = parser.parse_args()
  136. projects = yaml.safe_load(open(args.infile, 'r'))
  137. VALID_LABELS = ["acl-config", "cgit-alias", "description",
  138. "docimpact-group", "external-tracker-url", "groups",
  139. "homepage", "options", "project", "upstream",
  140. "upstream-prefix", "use-storyboard"]
  141. VALID_SCHEMES = ['https://', 'http://', 'git://']
  142. DESCRIPTION_REQUIRED = ['openstack', 'openstack-infra', 'openstack-dev',
  143. 'stackforge']
  144. VALID_OPTIONS = ['delay-release', 'track-upstream', 'translate']
  145. CGIT_ALIAS_SITES = ['zuul-ci.org']
  146. for p in projects:
  147. name = p.get('project')
  148. repo_group, repo_name = name.split('/')
  149. if not name:
  150. # not a project
  151. found_errors += 1
  152. print("ERROR: Entry is not a project %s" % p)
  153. continue
  154. if args.verbose:
  155. print('Checking %s' % name)
  156. description = p.get('description')
  157. # *very* simple check for common description mistakes
  158. badwords = (
  159. # (words), what_words_should_be
  160. (('openstack', 'Openstack', 'Open Stack'), 'OpenStack'),
  161. (('Devstack', 'devstack'), 'DevStack'),
  162. (('astor', 'Astor', 'astra', 'Astra', 'astara'), 'Astara')
  163. )
  164. if description:
  165. # newlines here mess up cgit "repo.desc
  166. if '\n' in description:
  167. found_errors += 1
  168. print("ERROR: Descriptions should not contain newlines:")
  169. print(' "%s"' % description)
  170. for words, should_be in badwords:
  171. for word in words:
  172. # look for the bad word hanging out on it's own. Only
  173. # trick is "\b" doesn't consider "-" or '.' as a
  174. # word-boundary, so ignore it if it looks like some
  175. # sort of job-description (e.g. "foo-devstack-bar") or
  176. # a url ("foo.openstack.org")
  177. if re.search(r'(?<![-.])\b%s\b' % word, description):
  178. print("ERROR: project %s, description '%s': "
  179. "contains wrong word '%s', it should be '%s'" %
  180. (name, description, word, should_be))
  181. found_errors += 1
  182. if not description and repo_group in DESCRIPTION_REQUIRED:
  183. found_errors += 1
  184. print("ERROR: Project %s has no description" % name)
  185. continue
  186. # Check upstream URL
  187. # Allow git:// and https:// URLs for importing upstream repositories,
  188. # but not git@
  189. upstream = p.get('upstream')
  190. if upstream and 'track-upstream' not in p.get('options', []):
  191. openstack_repo = 'https://opendev.org/%s' % name
  192. try:
  193. # Check to see if we have already imported the project into
  194. # OpenStack, if so skip checking upstream.
  195. check_repo(openstack_repo)
  196. except git.exc.GitCommandError:
  197. # We haven't imported the repo yet, make sure upstream is
  198. # valid.
  199. found_errors += check_repo(upstream)
  200. if upstream:
  201. for prefix in VALID_SCHEMES:
  202. if upstream.startswith(prefix):
  203. break
  204. else:
  205. found_errors += 1
  206. print('ERROR: Upstream URLs should use a scheme in %s, '
  207. 'found %s in %s' %
  208. (VALID_SCHEMES, p['upstream'], name))
  209. # Check for any wrong entries
  210. for entry in p:
  211. for label in VALID_LABELS:
  212. if entry == label:
  213. break
  214. else:
  215. found_errors += 1
  216. print("ERROR: Unknown keyword '%s' in project %s" %
  217. (entry, name))
  218. # Check for valid cgit aliases
  219. cgit_alias = p.get('cgit_alias')
  220. if cgit_alias:
  221. if not isinstance(cgit_alias, dict):
  222. found_errors += 1
  223. print("ERROR: cgit alias in project %s must be a dict" %
  224. (name,))
  225. else:
  226. if 'site' not in cgit_alias or 'path' not in cgit_alias:
  227. found_errors += 1
  228. print("ERROR: cgit alias in project %s must have "
  229. "a site and path" % (name,))
  230. else:
  231. site = cgit_alias['site']
  232. path = cgit_alias['path']
  233. if path.startswith('/'):
  234. found_errors += 1
  235. print("ERROR: cgit alias path in project %s must "
  236. "not begin with /" % (name,))
  237. if site not in CGIT_ALIAS_SITES:
  238. found_errors += 1
  239. print("ERROR: cgit alias site in project %s is "
  240. "not valid" % (name,))
  241. # Check for valid options
  242. for option in p.get('options', []):
  243. if option not in VALID_OPTIONS:
  244. found_errors += 1
  245. print("ERROR: Unknown option '%s' in project %s" %
  246. (option, name))
  247. # Check redundant acl-config
  248. acl_config = p.get('acl-config')
  249. if acl_config:
  250. if acl_config.endswith(name + '.config'):
  251. found_errors += 1
  252. print("ERROR: Project %s has redundant acl_config line, "
  253. "remove it." % name)
  254. if not acl_config.startswith('/home/gerrit2/acls/'):
  255. found_errors += 1
  256. print("ERROR: Project %s has wrong acl_config line, "
  257. "fix the path." % name)
  258. acl_file = os.path.join(args.acldir,
  259. acl_config[len('/home/gerrit2/acls/'):])
  260. if not os.path.isfile(acl_file):
  261. found_errors += 1
  262. print("ERROR: Project %s has non existing acl_config line" %
  263. name)
  264. else:
  265. # Check that default file exists
  266. acl_file = os.path.join(args.acldir, name + ".config")
  267. if not os.path.isfile(acl_file):
  268. found_errors += 1
  269. print("ERROR: Project %s has no default acl-config file" %
  270. name)
  271. # Check redundant groups entry:
  272. # By default the groups entry is repo_name, no need to add this.
  273. groups = p.get('groups')
  274. storyboard = p.get('use-storyboard', False)
  275. if (groups and len(groups) == 1 and groups[0] == repo_name
  276. and not storyboard):
  277. found_errors += 1
  278. print("ERROR: Non-StoryBoard project %s has default groups entry, "
  279. "remove it" % name)
  280. # Check that groups is a list
  281. groups = p.get('groups')
  282. if (groups and not isinstance(groups, list)):
  283. found_errors += 1
  284. print("Error: groups entry for project %s is not a list." % name)
  285. found_errors += check_zuul_main(args.zuul_main_file, projects)
  286. if found_errors:
  287. print("Found %d error(s) in %s" % (found_errors, args.infile))
  288. sys.exit(1)
  289. if __name__ == '__main__':
  290. main()