versions.yaml updater tool: verify that images & tags do exist
This patch allows the tool to verify that container images with specific tags equal to the HEAD git commit id's we reference, do really exist on quay.io repository and are available for download. Change-Id: I2205f4fa63a085d94ef3c0e9d2ff511665df43e6
This commit is contained in:
parent
46a1513a9b
commit
b537f2e15e
@ -23,12 +23,18 @@
|
|||||||
# 2) tags of container images listed in dict `image_repo_git_url` in the
|
# 2) tags of container images listed in dict `image_repo_git_url` in the
|
||||||
# code below
|
# code below
|
||||||
#
|
#
|
||||||
|
# In addition to that, the tool verifies that container images with
|
||||||
|
# specific tags equal to the HEAD git commit id's we reference, do really
|
||||||
|
# exist on quay.io repository and are available for download.
|
||||||
|
#
|
||||||
|
|
||||||
from functools import reduce
|
from functools import reduce
|
||||||
import argparse
|
import argparse
|
||||||
|
import datetime
|
||||||
import logging
|
import logging
|
||||||
import operator
|
import operator
|
||||||
import os
|
import os
|
||||||
|
import requests
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -54,7 +60,8 @@ image_repo_git_url = {
|
|||||||
'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard',
|
'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard',
|
||||||
'quay.io/airshipit/armada': 'https://git.openstack.org/openstack/airship-armada',
|
'quay.io/airshipit/armada': 'https://git.openstack.org/openstack/airship-armada',
|
||||||
'quay.io/airshipit/deckhand': 'https://git.openstack.org/openstack/airship-deckhand',
|
'quay.io/airshipit/deckhand': 'https://git.openstack.org/openstack/airship-deckhand',
|
||||||
'quay.io/airshipit/divingbell': 'https://git.openstack.org/openstack/airship-divingbell',
|
# yes, divingbell image is just Ubuntu 16.04 image, and we don't check it's tag
|
||||||
|
#'docker.io/ubuntu': 'https://git.openstack.org/openstack/airship-divingbell',
|
||||||
'quay.io/airshipit/drydock': 'https://git.openstack.org/openstack/airship-drydock',
|
'quay.io/airshipit/drydock': 'https://git.openstack.org/openstack/airship-drydock',
|
||||||
# maas-{rack,region}-controller images are built from airship-maas repository
|
# maas-{rack,region}-controller images are built from airship-maas repository
|
||||||
'quay.io/airshipit/maas-rack-controller': 'https://git.openstack.org/openstack/airship-maas',
|
'quay.io/airshipit/maas-rack-controller': 'https://git.openstack.org/openstack/airship-maas',
|
||||||
@ -65,17 +72,33 @@ image_repo_git_url = {
|
|||||||
# sstream-cache image is built from airship-maas repository
|
# sstream-cache image is built from airship-maas repository
|
||||||
'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas',
|
'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas',
|
||||||
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios'
|
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios'
|
||||||
# Disabled by Kaspars: https://review.openstack.org/#/c/596909/21/tools/updater.py@53
|
# FIXME (Kaspars): https://review.openstack.org/#/c/596909/21/tools/updater.py@53
|
||||||
#'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter'
|
#'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter'
|
||||||
}
|
}
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
# Temporary dict of git url's and cached commit id's: {'git_url': 'commit_id'}
|
# Temporary dict of git url's and cached commit id's: {'git_url': 'commit_id'}
|
||||||
|
global git_url_commit_ids
|
||||||
git_url_commit_ids = {}
|
git_url_commit_ids = {}
|
||||||
|
# Temporary dict of image repo's and status of image on quay.io
|
||||||
|
global image_repo_status
|
||||||
|
image_repo_status = {}
|
||||||
dict_path = None
|
dict_path = None
|
||||||
|
|
||||||
|
|
||||||
|
def inverse_dict(dic):
|
||||||
|
"""Accepts dictionary, returns dictionary where keys become values,
|
||||||
|
and values become keys"""
|
||||||
|
new_dict = {}
|
||||||
|
for k, v in dic.items():
|
||||||
|
new_dict[v] = k
|
||||||
|
return new_dict
|
||||||
|
|
||||||
|
|
||||||
|
git_url_image_repo = inverse_dict(image_repo_git_url)
|
||||||
|
|
||||||
|
|
||||||
# https://stackoverflow.com/a/35585837
|
# https://stackoverflow.com/a/35585837
|
||||||
def lsremote(url, remote_ref):
|
def lsremote(url, remote_ref):
|
||||||
"""Accepts git url and remote reference, returns git commit id."""
|
"""Accepts git url and remote reference, returns git commit id."""
|
||||||
@ -92,14 +115,70 @@ def get_commit_id(url):
|
|||||||
# If we don't have this git url in our url's dictionary,
|
# If we don't have this git url in our url's dictionary,
|
||||||
# fetch latest commit ID and add new dictionary entry
|
# fetch latest commit ID and add new dictionary entry
|
||||||
logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
|
logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
|
||||||
|
logging.debug('image_repo_status: %s', image_repo_status)
|
||||||
if url not in git_url_commit_ids:
|
if url not in git_url_commit_ids:
|
||||||
logging.debug('git url: ' + url +
|
logging.debug('git url: ' + url +
|
||||||
' is not in git_url_commit_ids dict;' +
|
' is not in git_url_commit_ids dict;' +
|
||||||
' adding it with HEAD commit id')
|
' adding it with HEAD commit id')
|
||||||
git_url_commit_ids[url] = lsremote(url, 'HEAD')
|
git_url_commit_ids[url] = lsremote(url, 'HEAD')
|
||||||
|
if url in git_url_image_repo:
|
||||||
|
if git_url_image_repo[url] not in image_repo_status:
|
||||||
|
image_repo_status[git_url_image_repo[url]] = \
|
||||||
|
verify_image_tag(git_url_image_repo[url], git_url_commit_ids[url])
|
||||||
|
else:
|
||||||
|
logging.debug('We checked image ' + git_url_image_repo[url] +
|
||||||
|
' on quay.io already, skipping')
|
||||||
return git_url_commit_ids[url]
|
return git_url_commit_ids[url]
|
||||||
|
|
||||||
|
|
||||||
|
def verify_image_tag(image, git_commit_id):
|
||||||
|
"""Verify if image with certain tag exists on quay.io,
|
||||||
|
returns 0 (image not hosted on quay.io), True, or False
|
||||||
|
"""
|
||||||
|
if not image.startswith('quay.io/'):
|
||||||
|
logging.info('Unable to verify if image ' + image + ':' +
|
||||||
|
git_commit_id + ' with this specific tag exists' +
|
||||||
|
' in containers repository: only quay.io is' +
|
||||||
|
' supported at the moment')
|
||||||
|
return 0
|
||||||
|
|
||||||
|
logging.info('Verifying if image ' + image + ':' + git_commit_id +
|
||||||
|
' with this specific tag exists on quay.io...')
|
||||||
|
|
||||||
|
hash_image = image.split('/')
|
||||||
|
res = requests.get('https://quay.io/api/v1/repository/' +
|
||||||
|
hash_image[1] + '/' + hash_image[2] +
|
||||||
|
'/tag/?specificTag=' + git_commit_id)
|
||||||
|
|
||||||
|
if res.status_code != 200:
|
||||||
|
logging.error('Image %s is not available on quay.io or it' +
|
||||||
|
' requires authentication', image)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
res = res.json()
|
||||||
|
|
||||||
|
for tag in res['tags']:
|
||||||
|
# Normally there should be only one tag description
|
||||||
|
if tag['name'] == git_commit_id:
|
||||||
|
if 'end_ts' not in tag:
|
||||||
|
# Active image tag
|
||||||
|
logging.info('Image ' + image + ':' + git_commit_id +
|
||||||
|
' with this specific tag corresponding to the' +
|
||||||
|
' commit id of the git HEAD exists on quay.io,' +
|
||||||
|
' last modified on ' +
|
||||||
|
datetime.datetime.fromtimestamp(tag['start_ts']).isoformat() +
|
||||||
|
' UTC')
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
# Tag used to exist
|
||||||
|
logging.error('Image ' + image + ':' + git_commit_id +
|
||||||
|
' with this specific tag no longer exists on quay.io')
|
||||||
|
return False
|
||||||
|
logging.error('There is no image ' + image + ':' + git_commit_id +
|
||||||
|
' with this specific tag on quay.io or it requires authentication')
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
# https://stackoverflow.com/a/14692747
|
# https://stackoverflow.com/a/14692747
|
||||||
def get_by_path(root, items):
|
def get_by_path(root, items):
|
||||||
"""Access a nested object in root by item sequence."""
|
"""Access a nested object in root by item sequence."""
|
||||||
|
Loading…
Reference in New Issue
Block a user