New option --out-file, various fixes in logic
1) added --out-file option 2) fixed imports for time and json 3) fixed retries logic to exit loop once we get valid response 4) fixed reference to an undefined variable Change-Id: Iebbb83079978eb22eb13e6c93642a35f5bd14a20
This commit is contained in:
parent
9ac56a7157
commit
0cf96f8b55
|
@ -18,24 +18,24 @@
|
||||||
# versions.yaml file updater tool
|
# versions.yaml file updater tool
|
||||||
#
|
#
|
||||||
# Being run in directory with versions.yaml, will create versions.new.yaml,
|
# Being run in directory with versions.yaml, will create versions.new.yaml,
|
||||||
# with updated git commit id's to the latest HEAD in:
|
# with updated git commit id's to the latest HEAD in references of all
|
||||||
# 1) references of all charts
|
# charts.
|
||||||
# 2) tags of container images listed in dict `image_repo_git_url` in the
|
|
||||||
# code below
|
|
||||||
#
|
#
|
||||||
# In addition to that, the tool verifies that container images with
|
# In addition to that, the tool updates references to the container images
|
||||||
# specific tags equal to the HEAD git commit id's we reference, do really
|
# with the tag, equal to the latest image which exists on quay.io
|
||||||
# exist on quay.io repository and are available for download.
|
# repository and is available for download.
|
||||||
#
|
#
|
||||||
|
|
||||||
from functools import reduce
|
|
||||||
import argparse
|
import argparse
|
||||||
import datetime
|
import datetime
|
||||||
|
from functools import reduce
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import operator
|
import operator
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import git
|
import git
|
||||||
|
@ -44,17 +44,23 @@ except ImportError as e:
|
||||||
sys.exit("Failed to import git/yaml libraries needed to run" +
|
sys.exit("Failed to import git/yaml libraries needed to run" +
|
||||||
"this tool %s" % str(e))
|
"this tool %s" % str(e))
|
||||||
|
|
||||||
descr_text="Process versions.yaml and create versions.new.yaml with \
|
descr_text="Being run in directory with versions.yaml, will create \
|
||||||
updated git commit id\'s to the latest HEAD in: \
|
versions.new.yaml, with updated git commit id's to the \
|
||||||
1) references of all charts \
|
latest HEAD in references of all charts. In addition to \
|
||||||
2) tags of container images listed in dict \
|
that, the tool updates references to the container images \
|
||||||
`image_repo_git_url` in the code."
|
with the tag, equal to the latest image which exists on \
|
||||||
|
quay.io repository and is available for download."
|
||||||
parser = argparse.ArgumentParser(description=descr_text)
|
parser = argparse.ArgumentParser(description=descr_text)
|
||||||
|
|
||||||
# Dictionary containing container image repository url to git url mapping
|
# Dictionary containing container image repository url to git url mapping
|
||||||
#
|
#
|
||||||
# We expect that each image in container image repository has image tag which
|
# We expect that each image in container image repository has image tag which
|
||||||
# equals to the git commit id of the HEAD in corresponding git repository
|
# equals to the git commit id of the HEAD in corresponding git repository.
|
||||||
|
#
|
||||||
|
# NOTE(roman_g): currently this is not the case, and image is built/tagged not
|
||||||
|
# on every merge, and there could be a few hours delay between merge and image
|
||||||
|
# re-built and published due to the OpenStack Foundation Zuul infrastructure
|
||||||
|
# being overloaded.
|
||||||
image_repo_git_url = {
|
image_repo_git_url = {
|
||||||
# airflow image is built from airship-shipyard repository
|
# airflow image is built from airship-shipyard repository
|
||||||
'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard',
|
'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard',
|
||||||
|
@ -129,9 +135,8 @@ def get_image_tag(image):
|
||||||
returns 0 (image not hosted on quay.io), True, or False
|
returns 0 (image not hosted on quay.io), True, or False
|
||||||
"""
|
"""
|
||||||
if not image.startswith('quay.io/'):
|
if not image.startswith('quay.io/'):
|
||||||
logging.info('Unable to verify if image ' + image + ':' +
|
logging.info('Unable to verify if image ' + image +
|
||||||
git_commit_id + ' with this specific tag exists' +
|
' is in containers repository: only quay.io is' +
|
||||||
' in containers repository: only quay.io is' +
|
|
||||||
' supported at the moment')
|
' supported at the moment')
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
@ -144,14 +149,15 @@ def get_image_tag(image):
|
||||||
url = 'https://quay.io/api/v1/repository/' + \
|
url = 'https://quay.io/api/v1/repository/' + \
|
||||||
hash_image[1] + '/' + hash_image[2] + '/tag'
|
hash_image[1] + '/' + hash_image[2] + '/tag'
|
||||||
|
|
||||||
while retries <= max_retries:
|
while retries < max_retries:
|
||||||
retries = retries +1
|
retries = retries + 1
|
||||||
try:
|
try:
|
||||||
res = requests.get(url, timeout = 5)
|
res = requests.get(url, timeout = 5)
|
||||||
|
if res.ok:
|
||||||
|
break
|
||||||
except requests.exceptions.Timeout:
|
except requests.exceptions.Timeout:
|
||||||
logging.warning("Failed to fetch url %s" % res.url)
|
logging.warning("Failed to fetch url %s" % res.url)
|
||||||
sleep(1)
|
time.sleep(1)
|
||||||
|
|
||||||
if retries == max_retries:
|
if retries == max_retries:
|
||||||
logging.error("Failed to connect to quay.io")
|
logging.error("Failed to connect to quay.io")
|
||||||
return 0
|
return 0
|
||||||
|
@ -162,7 +168,7 @@ def get_image_tag(image):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
res = res.json()
|
res = res.json()
|
||||||
except json.decoder.JSONDecodeError:
|
except json.decoder.JSONDecodeError: # pylint: disable=no-member
|
||||||
logging.error('Unable to parse response from quay.io (%s)' % res.url)
|
logging.error('Unable to parse response from quay.io (%s)' % res.url)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
@ -283,13 +289,18 @@ def traverse(obj, dict_path=None):
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
"""Small Main program"""
|
"""Small Main program"""
|
||||||
|
|
||||||
parser.add_argument('--in-file', default = 'versions.yaml',
|
parser.add_argument('--in-file', default='versions.yaml',
|
||||||
help = '/path/to/versions.yaml file')
|
help='/path/to/versions.yaml input file; default - "./versions.yaml"')
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
|
parser.add_argument('--out-file', default='versions.yaml',
|
||||||
|
help='name of output file; default - "versions.yaml" (overwrite existing)')
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
in_file = args.in_file
|
in_file = args.in_file
|
||||||
|
out_file = args.out_file
|
||||||
|
|
||||||
if os.path.isfile(in_file):
|
if os.path.isfile(in_file):
|
||||||
|
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), out_file)
|
||||||
with open(in_file, 'r') as f:
|
with open(in_file, 'r') as f:
|
||||||
f_old = f.read()
|
f_old = f.read()
|
||||||
versions_data_dict = yaml.safe_load(f_old)
|
versions_data_dict = yaml.safe_load(f_old)
|
||||||
|
@ -301,8 +312,9 @@ if __name__ == '__main__':
|
||||||
# Traverse loaded yaml and change it
|
# Traverse loaded yaml and change it
|
||||||
traverse(versions_data_dict)
|
traverse(versions_data_dict)
|
||||||
|
|
||||||
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), 'versions.new.yaml')
|
|
||||||
with open(out_file, 'w') as f:
|
with open(out_file, 'w') as f:
|
||||||
|
if os.path.samefile(in_file, out_file):
|
||||||
|
logging.info('Overwriting %s' % in_file)
|
||||||
f.write(yaml.safe_dump(versions_data_dict,
|
f.write(yaml.safe_dump(versions_data_dict,
|
||||||
default_flow_style=False,
|
default_flow_style=False,
|
||||||
explicit_end=True, explicit_start=True,
|
explicit_end=True, explicit_start=True,
|
||||||
|
|
Loading…
Reference in New Issue