airship
/
treasuremap
Code Issues Proposed changes

Add cache for results of requests to quay.io in Updater tool 

All changes are in Updater tool.
1) Cache results of requests to quay.io
2) Fix formatting of strings and strings in logs
3) Improve logging for http requests to quay.io
4) Fix failure when --out-file is specified with path
5) Fix quay.io HTTP 301 permanent redirect from '.../tag' to '.../tag/'

Change-Id: I7d5cec65c713be261af0997774ee945545b5090a
This commit is contained in:
Roman Gorshunov 2019-02-07 14:21:06 +01:00
parent 930e873ef5
commit 23b8e13c5b
1 changed files with 75 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
tools/updater.py
View File

@ -44,7 +44,7 @@ except ImportError as e:
sys.exit("Failed to import git/yaml libraries needed to run" + sys.exit("Failed to import git/yaml libraries needed to run" +
"this tool %s" % str(e)) "this tool %s" % str(e))
descr_text="Being run in directory with versions.yaml, will create \ descr_text = "Being run in directory with versions.yaml, will create \
versions.new.yaml, with updated git commit id's to the \ versions.new.yaml, with updated git commit id's to the \
latest HEAD in references of all charts. In addition to \ latest HEAD in references of all charts. In addition to \
that, the tool updates references to the container images \ that, the tool updates references to the container images \
@ -78,7 +78,8 @@ image_repo_git_url = {
# sstream-cache image is built from airship-maas repository # sstream-cache image is built from airship-maas repository
'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas', 'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas',
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios', 'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios',
'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter' 'quay.io/attcomdev/prometheus-openstack-exporter':
'https://github.com/att-comdev/prometheus-openstack-exporter'
} }
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
@ -129,7 +130,7 @@ def lsremote(url, remote_ref):
"""Accepts git url and remote reference, returns git commit id.""" """Accepts git url and remote reference, returns git commit id."""
git_commit_id_remote_ref = {} git_commit_id_remote_ref = {}
g = git.cmd.Git() g = git.cmd.Git()
logging.info('Fetching ' + url + ' ' + remote_ref + ' reference...') logging.info("Fetching %s %s reference...", url, remote_ref)
hash_ref_list = g.ls_remote(url, remote_ref).split('\t') hash_ref_list = g.ls_remote(url, remote_ref).split('\t')
git_commit_id_remote_ref[hash_ref_list[1]] = hash_ref_list[0] git_commit_id_remote_ref[hash_ref_list[1]] = hash_ref_list[0]
return git_commit_id_remote_ref[remote_ref] return git_commit_id_remote_ref[remote_ref]
@ -140,11 +141,10 @@ def get_commit_id(url):
# If we don't have this git url in our url's dictionary, # If we don't have this git url in our url's dictionary,
# fetch latest commit ID and add new dictionary entry # fetch latest commit ID and add new dictionary entry
logging.debug('git_url_commit_ids: %s', git_url_commit_ids) logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
logging.debug('image_repo_status: %s', image_repo_status)
if url not in git_url_commit_ids: if url not in git_url_commit_ids:
logging.debug('git url: ' + url + logging.debug("git url: %s" +
' is not in git_url_commit_ids dict;' + " is not in git_url_commit_ids dict;" +
' adding it with HEAD commit id') " adding it with HEAD commit id", url)
git_url_commit_ids[url] = lsremote(url, 'HEAD') git_url_commit_ids[url] = lsremote(url, 'HEAD')
return git_url_commit_ids[url] return git_url_commit_ids[url]
@ -155,41 +155,64 @@ def get_image_tag(image):
returns 0 (image not hosted on quay.io), True, or False returns 0 (image not hosted on quay.io), True, or False
""" """
if not image.startswith('quay.io/'): if not image.startswith('quay.io/'):
logging.info('Unable to verify if image ' + image + logging.info("Unable to verify if image %s" +
' is in containers repository: only quay.io is' + " is in containers repository: only quay.io is" +
' supported at the moment') " supported at the moment", image)
return 0 return 0
logging.info('Getting latest tag for image %s' % image) # If we don't have this image in our images's dictionary,
# fetch latest tag and add new dictionary entry
logging.debug('image_repo_status: %s', image_repo_status)
if image not in image_repo_status:
logging.debug("image: %s" +
" is not in image_repo_status dict;" +
" adding it with latest tag", image)
image_repo_status[image] = get_image_latest_tag(image)
retries = 0 return image_repo_status[image]
max_retries = 5
def get_image_latest_tag(image):
"""Get latest image tag from quay.io,
returns latest image tag string, or 0 if a problem occured.
"""
attempt = 0
max_attempts = 5
hash_image = image.split('/') hash_image = image.split('/')
url = 'https://quay.io/api/v1/repository/' + \ url = 'https://quay.io/api/v1/repository/{}/{}/tag/'
hash_image[1] + '/' + hash_image[2] + '/tag' url = url.format(hash_image[1], hash_image[2])
logging.info("Fetching latest tag for image %s (%s)...", image, url)
while retries < max_retries: while attempt < max_attempts:
retries = retries + 1 attempt = attempt + 1
try: try:
res = requests.get(url, timeout = 5) res = requests.get(url, timeout=5)
if res.ok: if res.ok:
break break
except requests.exceptions.Timeout: except requests.exceptions.Timeout:
logging.warning("Failed to fetch url %s" % res.url) logging.warning("Failed to fetch url %s for %d attempt(s)", url, attempt)
time.sleep(1) time.sleep(1)
if retries == max_retries: except requests.exceptions.TooManyRedirects:
logging.error("Failed to connect to quay.io") logging.error("Failed to fetch url %s, TooManyRedirects", url)
return 0
except requests.exceptions.RequestException as e:
logging.error("Failed to fetch url %s, error: %s", url, e)
return 0
if attempt == max_attempts:
logging.error("Failed to connect to quay.io for %d attempt(s)", attempt)
return 0 return 0
if res.status_code != 200: if res.status_code != 200:
logging.error('Image %s is not available on quay.io or ' + logging.error("Image %s is not available on quay.io or " +
'requires authentication', image) "requires authentication", image)
return 0
try: try:
res = res.json() res = res.json()
except json.decoder.JSONDecodeError: # pylint: disable=no-member except json.decoder.JSONDecodeError: # pylint: disable=no-member
logging.error('Unable to parse response from quay.io (%s)' % res.url) logging.error("Unable to parse response from quay.io (%s)", res.url)
return 0 return 0
try: try:
@ -198,10 +221,10 @@ def get_image_tag(image):
if tag['name'] != 'master' and tag['name'] != 'latest': if tag['name'] != 'master' and tag['name'] != 'latest':
return tag['name'] return tag['name']
except KeyError: except KeyError:
logging.error('Unable to parse response from quay.io (%s)' % res.url) logging.error("Unable to parse response from quay.io (%s)", res.url)
return 0 return 0
logging.error("Image with end_ts in path %s not found" % image) logging.error("Image with end_ts in path %s not found", image)
return 0 return 0
@ -248,17 +271,16 @@ def traverse(obj, dict_path=None):
# Update git commit id in reference field of dictionary # Update git commit id in reference field of dictionary
if old_git_commit_id != new_git_commit_id: if old_git_commit_id != new_git_commit_id:
logging.info('Updating git reference for chart %s from %s to ' + logging.info("Updating git reference for chart %s from %s to %s (%s)",
'%s (%s)',
k, old_git_commit_id, new_git_commit_id, k, old_git_commit_id, new_git_commit_id,
git_url) git_url)
v['reference'] = new_git_commit_id v['reference'] = new_git_commit_id
else: else:
logging.info('Git reference %s for chart %s is already up to date (%s) ', logging.info("Git reference %s for chart %s is already up to date (%s)",
old_git_commit_id, k, git_url) old_git_commit_id, k, git_url)
else: else:
logging.debug('value %s inside object is not a dictionary, or it does not ' + logging.debug("value %s inside object is not a dictionary, or it does not " +
'contain key \'type\' with value \'git\', skipping', v) "contain key \'type\' with value \'git\', skipping", v)
# Traverse one level deeper # Traverse one level deeper
traverse(v, dict_path + [k]) traverse(v, dict_path + [k])
@ -294,19 +316,19 @@ def traverse(obj, dict_path=None):
new_image_tag = get_image_tag(image) new_image_tag = get_image_tag(image)
if new_image_tag == 0: if new_image_tag == 0:
logging.error("Failed to get image tag for %s" % image) logging.error("Failed to get image tag for %s", image)
sys.exit(1) sys.exit(1)
# Update git commit id in tag of container image # Update git commit id in tag of container image
if old_image_tag != new_image_tag: if old_image_tag != new_image_tag:
logging.info('Updating git commit id in ' + logging.info("Updating git commit id in " +
'tag of container image %s from %s to %s', "tag of container image %s from %s to %s",
image, old_image_tag, new_image_tag) image, old_image_tag, new_image_tag)
set_by_path(versions_data_dict, dict_path, image + ':' + new_image_tag) set_by_path(versions_data_dict, dict_path, image + ':' + new_image_tag)
else: else:
logging.info('Git tag %s for container ' + logging.info("Git tag %s for container " +
'image %s is already up to date', "image %s is already up to date",
old_image_tag, image) old_image_tag, image)
else: else:
logging.debug('image_repo %s is not in %s string, skipping', image_repo, v) logging.debug('image_repo %s is not in %s string, skipping', image_repo, v)
@ -315,33 +337,42 @@ def traverse(obj, dict_path=None):
if __name__ == '__main__': if __name__ == '__main__':
"""Small Main program""" """Small Main program
"""
parser.add_argument('--in-file', default='versions.yaml', parser.add_argument('--in-file', default='versions.yaml',
help='/path/to/versions.yaml input file; default - "./versions.yaml"') help='/path/to/versions.yaml input file; default - "./versions.yaml"')
parser.add_argument('--out-file', default='versions.yaml', parser.add_argument('--out-file', default='versions.yaml',
help='name of output file; default - "versions.yaml" (overwrite existing)') help='name of output file; default - "versions.yaml" (overwrite existing)')
parser.add_argument('--skip', parser.add_argument('--skip',
help='comma-delimited list of images and charts to skip during the update') help='comma-delimited list of images and charts to skip during the update')
args = parser.parse_args() args = parser.parse_args()
in_file = args.in_file in_file = args.in_file
out_file = args.out_file out_file = args.out_file
if args.skip: if args.skip:
skip_list = tuple(args.skip.strip().split(",")) skip_list = tuple(args.skip.strip().split(","))
logging.info('Skip list: %s', skip_list) logging.info("Skip list: %s", skip_list)
else: else:
skip_list = None skip_list = None
if os.path.basename(out_file) != out_file:
logging.error("Name of the output file must not contain path, " +
"but only the file name.")
print("\n")
parser.print_help()
sys.exit(1)
if os.path.isfile(in_file): if os.path.isfile(in_file):
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), out_file) out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), out_file)
with open(in_file, 'r') as f: with open(in_file, 'r') as f:
f_old = f.read() f_old = f.read()
versions_data_dict = yaml.safe_load(f_old) versions_data_dict = yaml.safe_load(f_old)
else: else:
logging.error("Can\'t find versions.yaml file.\n") logging.error("Can\'t find versions.yaml file.")
print("\n")
parser.print_help() parser.print_help()
sys.exit(1) sys.exit(1)
@ -350,9 +381,9 @@ if __name__ == '__main__':
with open(out_file, 'w') as f: with open(out_file, 'w') as f:
if os.path.samefile(in_file, out_file): if os.path.samefile(in_file, out_file):
logging.info('Overwriting %s' % in_file) logging.info("Overwriting %s", in_file)
f.write(yaml.safe_dump(versions_data_dict, f.write(yaml.safe_dump(versions_data_dict,
default_flow_style=False, default_flow_style=False,
explicit_end=True, explicit_start=True, explicit_end=True, explicit_start=True,
width=4096)) width=4096))
logging.info('New versions.yaml created as %s' % out_file) logging.info("New versions.yaml created as %s", out_file)