Reference Airship manifests, CICD, and reference architecture.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

585 lines
20 KiB

  1. #!/usr/bin/env python3
  2. # Copyright 2018 AT&T Intellectual Property. All other rights reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. #
  16. # versions.yaml file updater tool
  17. #
  18. # Being run in directory with versions.yaml, will create versions.new.yaml,
  19. # with updated git commit id's to the latest HEAD in references of all
  20. # charts.
  21. #
  22. # In addition to that, the tool updates references to the container images
  23. # with the tag, equal to the latest image which exists on quay.io
  24. # repository and is available for download.
  25. #
  26. import argparse
  27. import copy
  28. import datetime
  29. from functools import reduce
  30. import json
  31. import logging
  32. import operator
  33. import os
  34. import requests
  35. import sys
  36. import time
  37. try:
  38. import git
  39. import yaml
  40. except ImportError as e:
  41. sys.exit(
  42. "Failed to import git/yaml libraries needed to run "
  43. "this tool %s" % str(e))
  44. descr_text = (
  45. "Being run in directory with versions.yaml, will create "
  46. "versions.new.yaml, with updated git commit id's to the "
  47. "latest HEAD in references of all charts. In addition to "
  48. "that, the tool updates references to the container images "
  49. "with the tag, equal to the latest image which exists on "
  50. "quay.io repository and is available for download.")
  51. parser = argparse.ArgumentParser(description=descr_text)
  52. # Dictionary containing container image repository url to git url mapping
  53. #
  54. # We expect that each image in container image repository has image tag which
  55. # equals to the git commit id of the HEAD in corresponding git repository.
  56. #
  57. # NOTE(roman_g): currently this is not the case, and image is built/tagged not
  58. # on every merge, and there could be a few hours delay between merge and image
  59. # re-built and published due to the OpenStack Foundation Zuul infrastructure
  60. # being overloaded.
  61. image_repo_git_url = {
  62. # airflow image is built from airship-shipyard repository
  63. "quay.io/airshipit/airflow": "https://opendev.org/airship/shipyard",
  64. "quay.io/airshipit/armada": "https://opendev.org/airship/armada",
  65. "quay.io/airshipit/deckhand": "https://opendev.org/airship/deckhand",
  66. # yes, divingbell image is just Ubuntu 16.04 image,
  67. # and we don't check it's tag:
  68. #"docker.io/ubuntu": "https://opendev.org/airship/divingbell",
  69. "quay.io/airshipit/drydock": "https://opendev.org/airship/drydock",
  70. # maas-{rack,region}-controller images are built
  71. # from airship-maas repository:
  72. "quay.io/airshipit/maas-rack-controller": "https://opendev.org/airship/maas",
  73. "quay.io/airshipit/maas-region-controller": "https://opendev.org/airship/maas",
  74. "quay.io/airshipit/pegleg": "https://opendev.org/airship/pegleg",
  75. "quay.io/airshipit/promenade": "https://opendev.org/airship/promenade",
  76. "quay.io/airshipit/shipyard": "https://opendev.org/airship/shipyard",
  77. # sstream-cache image is built from airship-maas repository
  78. "quay.io/airshipit/sstream-cache": "https://opendev.org/airship/maas"
  79. }
  80. logging.basicConfig(level=logging.INFO)
  81. LOG = logging.getLogger(__name__)
  82. # Dict of git url's and cached commit id's:
  83. # {"git_url": "commit_id"}
  84. git_url_commit_ids = {}
  85. # Dict of image repo's and (latest) tag of that image on quay.io:
  86. # {"image": "tag"}
  87. image_repo_tags = {}
  88. # Path in yaml dictionary
  89. dict_path = None
  90. def __represent_multiline_yaml_str():
  91. """Compel ``yaml`` library to use block style literals for multi-line
  92. strings to prevent unwanted multiple newlines.
  93. """
  94. yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str
  95. def repr_str(dumper, data):
  96. if "\n" in data:
  97. return dumper.represent_scalar(
  98. "tag:yaml.org,2002:str", data, style="|")
  99. return dumper.org_represent_str(data)
  100. yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
  101. __represent_multiline_yaml_str()
  102. def inverse_dict(dic):
  103. """Accepts dictionary, returns dictionary where keys become values,
  104. and values become keys"""
  105. new_dict = {}
  106. for k, v in dic.items():
  107. new_dict[v] = k
  108. return new_dict
  109. git_url_image_repo = inverse_dict(image_repo_git_url)
  110. # https://stackoverflow.com/a/35585837
  111. def lsremote(url, remote_ref):
  112. """Accepts git url and remote reference, returns git commit id."""
  113. git_commit_id_remote_ref = {}
  114. g = git.cmd.Git()
  115. LOG.info("Fetching %s %s reference...", url, remote_ref)
  116. hash_ref_list = g.ls_remote(url, remote_ref).split("\t")
  117. git_commit_id_remote_ref[hash_ref_list[1]] = hash_ref_list[0]
  118. return git_commit_id_remote_ref[remote_ref]
  119. def get_commit_id(url):
  120. """Accepts url of git repo and returns corresponding git commit hash"""
  121. # If we don't have this git url in our url's dictionary,
  122. # fetch latest commit ID and add new dictionary entry
  123. LOG.debug("git_url_commit_ids: %s", git_url_commit_ids)
  124. if url not in git_url_commit_ids:
  125. LOG.debug(
  126. "git url: %s " + "is not in git_url_commit_ids dict; "
  127. "adding it with HEAD commit id", url)
  128. git_url_commit_ids[url] = lsremote(url, "HEAD")
  129. return git_url_commit_ids[url]
  130. def get_image_tag(image):
  131. """Get latest image tag from quay.io,
  132. returns 0 (image not hosted on quay.io), True, or False
  133. """
  134. if not image.startswith("quay.io/"):
  135. LOG.info(
  136. "Unable to verify if image %s "
  137. "is in containers repository: only quay.io is "
  138. "supported at the moment", image)
  139. return 0
  140. # If we don't have this image in our images's dictionary,
  141. # fetch latest tag and add new dictionary entry
  142. LOG.debug("image_repo_tags: %s", image_repo_tags)
  143. if image not in image_repo_tags:
  144. LOG.debug(
  145. "image: %s " + "is not in image_repo_tags dict; "
  146. "adding it with latest tag", image)
  147. image_repo_tags[image] = get_image_latest_tag(image)
  148. return image_repo_tags[image]
  149. def get_image_latest_tag(image):
  150. """Get image tags from quay.io,
  151. returns latest image tag matching filter, or latest image tag if filter is not
  152. matched or not set, or 0 if a problem occured.
  153. """
  154. attempt = 0
  155. max_attempts = 10
  156. hash_image = image.split("/")
  157. url = "https://quay.io/api/v1/repository/{}/{}/tag/"
  158. url = url.format(hash_image[1], hash_image[2])
  159. LOG.info("Fetching tags for image %s (%s)...", image, url)
  160. while attempt < max_attempts:
  161. attempt += 1
  162. try:
  163. res = requests.get(url, timeout=5)
  164. if res.ok:
  165. break
  166. except requests.exceptions.Timeout:
  167. LOG.warning(
  168. "Failed to fetch url %s for %d/%d attempt(s)", url, attempt,
  169. max_attempts)
  170. time.sleep(5)
  171. except requests.exceptions.TooManyRedirects:
  172. logging.error("Failed to fetch url %s, TooManyRedirects", url)
  173. return 0
  174. except requests.exceptions.RequestException as e:
  175. logging.error("Failed to fetch url %s, error: %s", url, e)
  176. return 0
  177. if attempt == max_attempts:
  178. logging.error(
  179. "Failed to connect to quay.io for %d attempt(s)", attempt)
  180. return 0
  181. if res.status_code != 200:
  182. logging.error(
  183. "Image %s is not available on quay.io or "
  184. "requires authentication", image)
  185. return 0
  186. try:
  187. res = res.json()
  188. except json.decoder.JSONDecodeError: # pylint: disable=no-member
  189. logging.error("Unable to parse response from quay.io (%s)", res.url)
  190. return 0
  191. try:
  192. possible_tag = None
  193. for tag in res["tags"]:
  194. # skip images which are old (have "end_ts"), and
  195. # skip images tagged with "*latest*" or "*master*"
  196. if "end_ts" in tag or \
  197. any(i in tag["name"] for i in ("latest", "master")):
  198. continue
  199. # simply return first found tag is we don't have filter set
  200. if not tag_filter:
  201. return tag["name"]
  202. # return tag matching filter, if we have filter set
  203. if tag_filter in tag["name"]:
  204. return tag["name"]
  205. LOG.info(
  206. "Skipping tag %s as not matching to the filter %s",
  207. tag["name"], tag_filter)
  208. if not possible_tag:
  209. possible_tag = tag["name"]
  210. if possible_tag:
  211. LOG.info("Couldn't find better tag than %s", possible_tag)
  212. return possible_tag
  213. except KeyError:
  214. logging.error("Unable to parse response from quay.io (%s)", res.url)
  215. return 0
  216. logging.error("Image without end_ts in path %s not found", image)
  217. return 0
  218. # https://stackoverflow.com/a/14692747
  219. def get_by_path(root, items):
  220. """Access a nested object in root by item sequence."""
  221. return reduce(operator.getitem, items, root)
  222. def set_by_path(root, items, value):
  223. """Set a value in a nested object in root by item sequence."""
  224. get_by_path(root, items[:-1])[items[-1]] = value
  225. # Based on http://nvie.com/posts/modifying-deeply-nested-structures/
  226. def traverse(obj, dict_path=None):
  227. """Accepts Python dictionary with values.yaml contents,
  228. updates it with latest git commit id's.
  229. """
  230. LOG.debug(
  231. "traverse: dict_path: %s, object type: %s, object: %s", dict_path,
  232. type(obj), obj)
  233. if dict_path is None:
  234. dict_path = []
  235. if isinstance(obj, dict):
  236. # It's a dictionary element
  237. LOG.debug("this object is a dictionary")
  238. for k, v in obj.items():
  239. # If value v we are checking is a dictionary itself, and this
  240. # dictionary contains key named "type", and a value of key "type"
  241. # equals "git", then
  242. if isinstance(v, dict) and "type" in v and v["type"] == "git":
  243. old_git_commit_id = v["reference"]
  244. git_url = v["location"]
  245. if skip_list and k in skip_list:
  246. LOG.info(
  247. "Ignoring chart %s, it is in a "
  248. "skip list (%s)", k, git_url)
  249. continue
  250. new_git_commit_id = get_commit_id(git_url)
  251. # Update git commit id in reference field of dictionary
  252. if old_git_commit_id != new_git_commit_id:
  253. LOG.info(
  254. "Updating git reference for "
  255. "chart %s from %s to %s (%s)", k, old_git_commit_id,
  256. new_git_commit_id, git_url)
  257. v["reference"] = new_git_commit_id
  258. else:
  259. LOG.info(
  260. "Git reference %s for chart %s is already "
  261. "up to date (%s)", old_git_commit_id, k, git_url)
  262. else:
  263. LOG.debug(
  264. "value %s inside object is not a dictionary, "
  265. "or it does not contain key \"type\" with "
  266. "value \"git\", skipping", v)
  267. # Traverse one level deeper
  268. traverse(v, dict_path + [k])
  269. elif isinstance(obj, list):
  270. # It's a list element
  271. LOG.debug("this object is a list")
  272. for elem in obj:
  273. # TODO: Do we have any git references or container image tags in
  274. # versions.yaml which are inside lists? Probably not.
  275. traverse(elem, dict_path + [[]])
  276. else:
  277. # It's already a value
  278. LOG.debug("this object is a value")
  279. v = obj
  280. # Searching for container image repositories, we are only intrested in
  281. # strings; there could also be booleans or other types
  282. # we are not interested in.
  283. if isinstance(v, str):
  284. for image_repo in image_repo_git_url:
  285. if image_repo in v:
  286. LOG.debug("image_repo %s is in %s string", image_repo, v)
  287. # hash_v: {"&whatever repo_url", "git commit id tag"}
  288. # Note: "image" below could contain not just image,
  289. # but also "&ref host.domain/path/image"
  290. hash_v = v.split(":")
  291. image, old_image_tag = hash_v
  292. if skip_list and image in skip_list:
  293. LOG.info(
  294. "Ignoring image %s, it is in a "
  295. "skip list", image)
  296. continue
  297. new_image_tag = get_image_tag(image)
  298. if new_image_tag == 0:
  299. logging.error("Failed to get image tag for %s", image)
  300. sys.exit(1)
  301. # Update git commit id in tag of container image
  302. if old_image_tag != new_image_tag:
  303. LOG.info(
  304. "Updating git commit id in "
  305. "tag of container image %s from %s to %s", image,
  306. old_image_tag, new_image_tag)
  307. set_by_path(
  308. versions_data_dict, dict_path,
  309. image + ":" + new_image_tag)
  310. else:
  311. LOG.info(
  312. "Git tag %s for container "
  313. "image %s is already up to date", old_image_tag,
  314. image)
  315. else:
  316. LOG.debug(
  317. "image_repo %s is not in %s string, "
  318. "skipping", image_repo, v)
  319. else:
  320. LOG.debug("value %s is not string, skipping", v)
  321. def debug_dicts():
  322. """Print varioud dictionary contents on debug"""
  323. LOG.debug("git_url_commit_ids: %s", git_url_commit_ids)
  324. LOG.debug("image_repo_tags: %s", image_repo_tags)
  325. LOG.debug("image_repo_git_url: %s", image_repo_git_url)
  326. def print_versions_table():
  327. """Prints overall Git and images versions table."""
  328. debug_dicts()
  329. table_format = "{:48s} {:60s} {:54s} {:41s}\n"
  330. table_content = "\n"
  331. table_content += table_format.format(
  332. "Image repo", "Git repo", "Image repo tag", "Git repo Commit ID")
  333. # Copy dicts for later modification
  334. image_repo_tags_copy = copy.deepcopy(image_repo_tags)
  335. git_url_commit_ids_copy = copy.deepcopy(git_url_commit_ids)
  336. # Loop over
  337. # image_repo_git_url ({"image_repo", "git_repo"}),
  338. # and verify it's contents against the copies of
  339. # git_url_commit_ids ({"git_repo": "commit_id"})
  340. # and
  341. # image_repo_tags ({"image_repo": "tag"})
  342. # dictionaries
  343. for image_repo in image_repo_git_url:
  344. git_repo = image_repo_git_url[image_repo]
  345. if not image_repo in image_repo_tags_copy:
  346. image_repo_tags_copy[image_repo] = get_image_latest_tag(image_repo)
  347. if not git_repo in git_url_commit_ids_copy:
  348. git_url_commit_ids_copy[git_repo] = lsremote(git_repo, "HEAD")
  349. table_content += table_format.format(
  350. image_repo, git_repo, image_repo_tags_copy[image_repo],
  351. git_url_commit_ids_copy[git_repo])
  352. LOG.info("")
  353. for line in table_content.splitlines():
  354. LOG.info(line)
  355. LOG.info("")
  356. def print_missing_references():
  357. """Prints possibly missing references in versions.yaml."""
  358. debug_dicts()
  359. missing_references = {}
  360. # Loop over
  361. # image_repo_git_url ({"image_repo", "git_repo"}),
  362. # and verify it's contents against the contents of
  363. # git_url_commit_ids ({"git_repo": "commit_id"})
  364. # and
  365. # image_repo_tags ({"image_repo": "tag"})
  366. # dictionaries
  367. for image_repo in image_repo_git_url:
  368. git_repo = image_repo_git_url[image_repo]
  369. if not image_repo in image_repo_tags:
  370. missing_references[image_repo] = \
  371. image_repo + " is not in image_repo_tags"
  372. if not git_repo in git_url_commit_ids:
  373. missing_references[git_repo] = \
  374. git_repo + " is not in git_url_commit_ids"
  375. if missing_references:
  376. LOG.warning("")
  377. LOG.warning("Missing references:")
  378. for ref in missing_references:
  379. LOG.warning(missing_references[ref])
  380. LOG.warning("")
  381. LOG.warning(
  382. "Refs which are not in git_url_commit_ids mean that "
  383. "we have not been updating chart references (or "
  384. "there are no charts referred in versions.yaml)")
  385. LOG.warning(
  386. "Refs which are not in image_repo_tags mean that we "
  387. "have not been updating image tags (or there are no "
  388. "images referred in versions.yaml)")
  389. LOG.warning("")
  390. def print_outdated_images():
  391. """Print Git <-> image tag mismatches."""
  392. debug_dicts()
  393. possibly_outdated_images = []
  394. # Copy dicts for later modification
  395. image_repo_tags_copy = copy.deepcopy(image_repo_tags)
  396. git_url_commit_ids_copy = copy.deepcopy(git_url_commit_ids)
  397. # Loop over
  398. # image_repo_git_url ({"image_repo", "git_repo"}),
  399. # and verify it's contents against the contents of
  400. # git_url_commit_ids ({"git_repo": "commit_id"})
  401. # and
  402. # image_repo_tags ({"image_repo": "tag"})
  403. # dictionaries
  404. for image_repo in image_repo_git_url:
  405. git_repo = image_repo_git_url[image_repo]
  406. if not image_repo in image_repo_tags_copy:
  407. image_repo_tags_copy[image_repo] = get_image_latest_tag(image_repo)
  408. if not git_repo in git_url_commit_ids_copy:
  409. git_url_commit_ids_copy[git_repo] = lsremote(git_repo, "HEAD")
  410. # This is where we check if there is tag matching commit_id exists,
  411. # and if not, then we append that image_repo to the list of
  412. # possibly outdated images
  413. if git_url_commit_ids_copy[git_repo] not in image_repo_tags_copy[
  414. image_repo]:
  415. possibly_outdated_images.append(image_repo)
  416. if possibly_outdated_images:
  417. LOG.warning("")
  418. LOG.warning("Possibly outdated images:")
  419. for image in possibly_outdated_images:
  420. LOG.warning(image)
  421. LOG.warning("")
  422. if __name__ == "__main__":
  423. """Main program
  424. """
  425. parser.add_argument(
  426. "--in-file",
  427. default="versions.yaml",
  428. help="/path/to/versions.yaml input file; "
  429. "default - \"./versions.yaml\"")
  430. parser.add_argument(
  431. "--out-file",
  432. default="versions.yaml",
  433. help="name of output file; default - "
  434. "\"versions.yaml\" (overwrite existing)")
  435. parser.add_argument(
  436. "--skip",
  437. help="comma-delimited list of images and charts "
  438. "to skip during the update; e.g. \"ceph\" "
  439. "will skip all charts and images which have "
  440. "\"ceph\" in the name")
  441. parser.add_argument(
  442. '--tag-filter',
  443. help="e.g. \"ubuntu\"; update would use image ref. "
  444. "tags on quay.io matching the filter")
  445. args = parser.parse_args()
  446. in_file = args.in_file
  447. out_file = args.out_file
  448. if args.skip:
  449. skip_list = tuple(args.skip.strip().split(","))
  450. LOG.info("Skip list: %s", skip_list)
  451. else:
  452. skip_list = None
  453. tag_filter = args.tag_filter
  454. LOG.info("Tag filter: %s", tag_filter)
  455. if os.path.basename(out_file) != out_file:
  456. logging.error(
  457. "Name of the output file must not contain path, "
  458. "but only the file name.")
  459. print("\n")
  460. parser.print_help()
  461. sys.exit(1)
  462. if os.path.isfile(in_file):
  463. out_file = os.path.join(
  464. os.path.dirname(os.path.abspath(in_file)), out_file)
  465. with open(in_file, "r") as f:
  466. f_old = f.read()
  467. versions_data_dict = yaml.safe_load(f_old)
  468. else:
  469. logging.error("Can't find versions.yaml file.")
  470. print("\n")
  471. parser.print_help()
  472. sys.exit(1)
  473. # Traverse loaded yaml and change it
  474. traverse(versions_data_dict)
  475. print_versions_table()
  476. print_missing_references()
  477. print_outdated_images()
  478. with open(out_file, "w") as f:
  479. if os.path.samefile(in_file, out_file):
  480. LOG.info("Overwriting %s", in_file)
  481. f.write(
  482. yaml.safe_dump(
  483. versions_data_dict,
  484. default_flow_style=False,
  485. explicit_end=True,
  486. explicit_start=True,
  487. width=4096))
  488. LOG.info("New versions.yaml created as %s", out_file)