diff --git a/tools/updater.py b/tools/updater.py index 946edbc0a..a7e37e035 100755 --- a/tools/updater.py +++ b/tools/updater.py @@ -23,12 +23,18 @@ # 2) tags of container images listed in dict `image_repo_git_url` in the # code below # +# In addition to that, the tool verifies that container images with +# specific tags equal to the HEAD git commit id's we reference, do really +# exist on quay.io repository and are available for download. +# from functools import reduce import argparse +import datetime import logging import operator import os +import requests import sys try: @@ -54,7 +60,8 @@ image_repo_git_url = { 'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard', 'quay.io/airshipit/armada': 'https://git.openstack.org/openstack/airship-armada', 'quay.io/airshipit/deckhand': 'https://git.openstack.org/openstack/airship-deckhand', - 'quay.io/airshipit/divingbell': 'https://git.openstack.org/openstack/airship-divingbell', + # yes, divingbell image is just Ubuntu 16.04 image, and we don't check it's tag + #'docker.io/ubuntu': 'https://git.openstack.org/openstack/airship-divingbell', 'quay.io/airshipit/drydock': 'https://git.openstack.org/openstack/airship-drydock', # maas-{rack,region}-controller images are built from airship-maas repository 'quay.io/airshipit/maas-rack-controller': 'https://git.openstack.org/openstack/airship-maas', @@ -64,18 +71,33 @@ image_repo_git_url = { 'quay.io/airshipit/shipyard': 'https://git.openstack.org/openstack/airship-shipyard', # sstream-cache image is built from airship-maas repository 'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas', - 'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios' - # Disabled by Kaspars: https://review.openstack.org/#/c/596909/21/tools/updater.py@53 - #'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter' + 'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios', + 'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter' } logging.basicConfig(level=logging.INFO) # Temporary dict of git url's and cached commit id's: {'git_url': 'commit_id'} +global git_url_commit_ids git_url_commit_ids = {} +# Temporary dict of image repo's and status of image on quay.io +global image_repo_status +image_repo_status = {} dict_path = None +def inverse_dict(dic): + """Accepts dictionary, returns dictionary where keys become values, + and values become keys""" + new_dict = {} + for k, v in dic.items(): + new_dict[v] = k + return new_dict + + +git_url_image_repo = inverse_dict(image_repo_git_url) + + # https://stackoverflow.com/a/35585837 def lsremote(url, remote_ref): """Accepts git url and remote reference, returns git commit id.""" @@ -92,14 +114,92 @@ def get_commit_id(url): # If we don't have this git url in our url's dictionary, # fetch latest commit ID and add new dictionary entry logging.debug('git_url_commit_ids: %s', git_url_commit_ids) + logging.debug('image_repo_status: %s', image_repo_status) if url not in git_url_commit_ids: logging.debug('git url: ' + url + ' is not in git_url_commit_ids dict;' + ' adding it with HEAD commit id') git_url_commit_ids[url] = lsremote(url, 'HEAD') + if url in git_url_image_repo: + if git_url_image_repo[url] not in image_repo_status: + image_repo_status[git_url_image_repo[url]] = \ + verify_image_tag(git_url_image_repo[url], git_url_commit_ids[url]) + else: + logging.debug('We checked image ' + git_url_image_repo[url] + + ' on quay.io already, skipping') return git_url_commit_ids[url] +def verify_image_tag(image, git_commit_id): + """Verify if image with certain tag exists on quay.io, + returns 0 (image not hosted on quay.io), True, or False + """ + if not image.startswith('quay.io/'): + logging.info('Unable to verify if image ' + image + ':' + + git_commit_id + ' with this specific tag exists' + + ' in containers repository: only quay.io is' + + ' supported at the moment') + return 0 + + logging.info('Verifying if image ' + image + ':' + git_commit_id + + ' with this specific tag exists on quay.io...') + + retries = 0 + max_retries = 5 + + payload = {'specificTag': git_commit_id} + hash_image = image.split('/') + url = 'https://quay.io/api/v1/repository/' + \ + hash_image[1] + '/' + hash_image[2] + '/tag/' + + while retries <= max_retries: + retries = retries +1 + try: + res = requests.get(url, params = payload, timeout = 5) + except requests.exceptions.Timeout: + logging.warning("Failed to fetch url %s" % res.url) + sleep(1) + if retries == max_retries: + logging.error("Failed to connect to quay.io") + return 0 + + if res.status_code != 200: + logging.error('Image %s is not available on quay.io or it' + + ' requires authentication', image) + return 0 + + try: + res = res.json() + except json.decoder.JSONDecodeError: + logging.error('Unable to parse response from quay.io (%s)' % res.url) + return 0 + + try: + for tag in res['tags']: + # Normally there should be only one tag description + if tag['name'] == git_commit_id: + if 'end_ts' not in tag: + # Active image tag + logging.info('Image ' + image + ':' + git_commit_id + + ' with this specific tag corresponding to the' + + ' commit id of the git HEAD exists on quay.io,' + + ' last modified on ' + + datetime.datetime.fromtimestamp(tag['start_ts']).isoformat() + + ' UTC') + return True + else: + # Tag used to exist + logging.error('Image ' + image + ':' + git_commit_id + + ' with this specific tag no longer exists on quay.io') + return False + except KeyError: + logging.error('Unable to parse response from quay.io (%s)' % res.url) + return 0 + logging.error('There is no image ' + image + ':' + git_commit_id + + ' with this specific tag on quay.io or it requires authentication') + return False + + # https://stackoverflow.com/a/14692747 def get_by_path(root, items): """Access a nested object in root by item sequence."""