Merge "Add cache for results of requests to quay.io in Updater tool"
This commit is contained in:
commit
82b2e57147
119
tools/updater.py
119
tools/updater.py
|
@ -44,7 +44,7 @@ except ImportError as e:
|
||||||
sys.exit("Failed to import git/yaml libraries needed to run" +
|
sys.exit("Failed to import git/yaml libraries needed to run" +
|
||||||
"this tool %s" % str(e))
|
"this tool %s" % str(e))
|
||||||
|
|
||||||
descr_text="Being run in directory with versions.yaml, will create \
|
descr_text = "Being run in directory with versions.yaml, will create \
|
||||||
versions.new.yaml, with updated git commit id's to the \
|
versions.new.yaml, with updated git commit id's to the \
|
||||||
latest HEAD in references of all charts. In addition to \
|
latest HEAD in references of all charts. In addition to \
|
||||||
that, the tool updates references to the container images \
|
that, the tool updates references to the container images \
|
||||||
|
@ -78,7 +78,8 @@ image_repo_git_url = {
|
||||||
# sstream-cache image is built from airship-maas repository
|
# sstream-cache image is built from airship-maas repository
|
||||||
'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas',
|
'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas',
|
||||||
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios',
|
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios',
|
||||||
'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter'
|
'quay.io/attcomdev/prometheus-openstack-exporter':
|
||||||
|
'https://github.com/att-comdev/prometheus-openstack-exporter'
|
||||||
}
|
}
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
@ -129,7 +130,7 @@ def lsremote(url, remote_ref):
|
||||||
"""Accepts git url and remote reference, returns git commit id."""
|
"""Accepts git url and remote reference, returns git commit id."""
|
||||||
git_commit_id_remote_ref = {}
|
git_commit_id_remote_ref = {}
|
||||||
g = git.cmd.Git()
|
g = git.cmd.Git()
|
||||||
logging.info('Fetching ' + url + ' ' + remote_ref + ' reference...')
|
logging.info("Fetching %s %s reference...", url, remote_ref)
|
||||||
hash_ref_list = g.ls_remote(url, remote_ref).split('\t')
|
hash_ref_list = g.ls_remote(url, remote_ref).split('\t')
|
||||||
git_commit_id_remote_ref[hash_ref_list[1]] = hash_ref_list[0]
|
git_commit_id_remote_ref[hash_ref_list[1]] = hash_ref_list[0]
|
||||||
return git_commit_id_remote_ref[remote_ref]
|
return git_commit_id_remote_ref[remote_ref]
|
||||||
|
@ -140,11 +141,10 @@ def get_commit_id(url):
|
||||||
# If we don't have this git url in our url's dictionary,
|
# If we don't have this git url in our url's dictionary,
|
||||||
# fetch latest commit ID and add new dictionary entry
|
# fetch latest commit ID and add new dictionary entry
|
||||||
logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
|
logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
|
||||||
logging.debug('image_repo_status: %s', image_repo_status)
|
|
||||||
if url not in git_url_commit_ids:
|
if url not in git_url_commit_ids:
|
||||||
logging.debug('git url: ' + url +
|
logging.debug("git url: %s" +
|
||||||
' is not in git_url_commit_ids dict;' +
|
" is not in git_url_commit_ids dict;" +
|
||||||
' adding it with HEAD commit id')
|
" adding it with HEAD commit id", url)
|
||||||
git_url_commit_ids[url] = lsremote(url, 'HEAD')
|
git_url_commit_ids[url] = lsremote(url, 'HEAD')
|
||||||
|
|
||||||
return git_url_commit_ids[url]
|
return git_url_commit_ids[url]
|
||||||
|
@ -155,41 +155,64 @@ def get_image_tag(image):
|
||||||
returns 0 (image not hosted on quay.io), True, or False
|
returns 0 (image not hosted on quay.io), True, or False
|
||||||
"""
|
"""
|
||||||
if not image.startswith('quay.io/'):
|
if not image.startswith('quay.io/'):
|
||||||
logging.info('Unable to verify if image ' + image +
|
logging.info("Unable to verify if image %s" +
|
||||||
' is in containers repository: only quay.io is' +
|
" is in containers repository: only quay.io is" +
|
||||||
' supported at the moment')
|
" supported at the moment", image)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
logging.info('Getting latest tag for image %s' % image)
|
# If we don't have this image in our images's dictionary,
|
||||||
|
# fetch latest tag and add new dictionary entry
|
||||||
|
logging.debug('image_repo_status: %s', image_repo_status)
|
||||||
|
if image not in image_repo_status:
|
||||||
|
logging.debug("image: %s" +
|
||||||
|
" is not in image_repo_status dict;" +
|
||||||
|
" adding it with latest tag", image)
|
||||||
|
image_repo_status[image] = get_image_latest_tag(image)
|
||||||
|
|
||||||
retries = 0
|
return image_repo_status[image]
|
||||||
max_retries = 5
|
|
||||||
|
|
||||||
|
def get_image_latest_tag(image):
|
||||||
|
"""Get latest image tag from quay.io,
|
||||||
|
returns latest image tag string, or 0 if a problem occured.
|
||||||
|
"""
|
||||||
|
|
||||||
|
attempt = 0
|
||||||
|
max_attempts = 5
|
||||||
|
|
||||||
hash_image = image.split('/')
|
hash_image = image.split('/')
|
||||||
url = 'https://quay.io/api/v1/repository/' + \
|
url = 'https://quay.io/api/v1/repository/{}/{}/tag/'
|
||||||
hash_image[1] + '/' + hash_image[2] + '/tag'
|
url = url.format(hash_image[1], hash_image[2])
|
||||||
|
logging.info("Fetching latest tag for image %s (%s)...", image, url)
|
||||||
|
|
||||||
while retries < max_retries:
|
while attempt < max_attempts:
|
||||||
retries = retries + 1
|
attempt = attempt + 1
|
||||||
try:
|
try:
|
||||||
res = requests.get(url, timeout = 5)
|
res = requests.get(url, timeout=5)
|
||||||
if res.ok:
|
if res.ok:
|
||||||
break
|
break
|
||||||
except requests.exceptions.Timeout:
|
except requests.exceptions.Timeout:
|
||||||
logging.warning("Failed to fetch url %s" % res.url)
|
logging.warning("Failed to fetch url %s for %d attempt(s)", url, attempt)
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
if retries == max_retries:
|
except requests.exceptions.TooManyRedirects:
|
||||||
logging.error("Failed to connect to quay.io")
|
logging.error("Failed to fetch url %s, TooManyRedirects", url)
|
||||||
|
return 0
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
logging.error("Failed to fetch url %s, error: %s", url, e)
|
||||||
|
return 0
|
||||||
|
if attempt == max_attempts:
|
||||||
|
logging.error("Failed to connect to quay.io for %d attempt(s)", attempt)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
if res.status_code != 200:
|
if res.status_code != 200:
|
||||||
logging.error('Image %s is not available on quay.io or ' +
|
logging.error("Image %s is not available on quay.io or " +
|
||||||
'requires authentication', image)
|
"requires authentication", image)
|
||||||
|
return 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
res = res.json()
|
res = res.json()
|
||||||
except json.decoder.JSONDecodeError: # pylint: disable=no-member
|
except json.decoder.JSONDecodeError: # pylint: disable=no-member
|
||||||
logging.error('Unable to parse response from quay.io (%s)' % res.url)
|
logging.error("Unable to parse response from quay.io (%s)", res.url)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -198,10 +221,10 @@ def get_image_tag(image):
|
||||||
if tag['name'] != 'master' and tag['name'] != 'latest':
|
if tag['name'] != 'master' and tag['name'] != 'latest':
|
||||||
return tag['name']
|
return tag['name']
|
||||||
except KeyError:
|
except KeyError:
|
||||||
logging.error('Unable to parse response from quay.io (%s)' % res.url)
|
logging.error("Unable to parse response from quay.io (%s)", res.url)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
logging.error("Image with end_ts in path %s not found" % image)
|
logging.error("Image with end_ts in path %s not found", image)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
@ -248,17 +271,16 @@ def traverse(obj, dict_path=None):
|
||||||
|
|
||||||
# Update git commit id in reference field of dictionary
|
# Update git commit id in reference field of dictionary
|
||||||
if old_git_commit_id != new_git_commit_id:
|
if old_git_commit_id != new_git_commit_id:
|
||||||
logging.info('Updating git reference for chart %s from %s to ' +
|
logging.info("Updating git reference for chart %s from %s to %s (%s)",
|
||||||
'%s (%s)',
|
|
||||||
k, old_git_commit_id, new_git_commit_id,
|
k, old_git_commit_id, new_git_commit_id,
|
||||||
git_url)
|
git_url)
|
||||||
v['reference'] = new_git_commit_id
|
v['reference'] = new_git_commit_id
|
||||||
else:
|
else:
|
||||||
logging.info('Git reference %s for chart %s is already up to date (%s) ',
|
logging.info("Git reference %s for chart %s is already up to date (%s)",
|
||||||
old_git_commit_id, k, git_url)
|
old_git_commit_id, k, git_url)
|
||||||
else:
|
else:
|
||||||
logging.debug('value %s inside object is not a dictionary, or it does not ' +
|
logging.debug("value %s inside object is not a dictionary, or it does not " +
|
||||||
'contain key \'type\' with value \'git\', skipping', v)
|
"contain key \'type\' with value \'git\', skipping", v)
|
||||||
|
|
||||||
# Traverse one level deeper
|
# Traverse one level deeper
|
||||||
traverse(v, dict_path + [k])
|
traverse(v, dict_path + [k])
|
||||||
|
@ -294,19 +316,19 @@ def traverse(obj, dict_path=None):
|
||||||
|
|
||||||
new_image_tag = get_image_tag(image)
|
new_image_tag = get_image_tag(image)
|
||||||
if new_image_tag == 0:
|
if new_image_tag == 0:
|
||||||
logging.error("Failed to get image tag for %s" % image)
|
logging.error("Failed to get image tag for %s", image)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# Update git commit id in tag of container image
|
# Update git commit id in tag of container image
|
||||||
if old_image_tag != new_image_tag:
|
if old_image_tag != new_image_tag:
|
||||||
logging.info('Updating git commit id in ' +
|
logging.info("Updating git commit id in " +
|
||||||
'tag of container image %s from %s to %s',
|
"tag of container image %s from %s to %s",
|
||||||
image, old_image_tag, new_image_tag)
|
image, old_image_tag, new_image_tag)
|
||||||
set_by_path(versions_data_dict, dict_path, image + ':' + new_image_tag)
|
set_by_path(versions_data_dict, dict_path, image + ':' + new_image_tag)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
logging.info('Git tag %s for container ' +
|
logging.info("Git tag %s for container " +
|
||||||
'image %s is already up to date',
|
"image %s is already up to date",
|
||||||
old_image_tag, image)
|
old_image_tag, image)
|
||||||
else:
|
else:
|
||||||
logging.debug('image_repo %s is not in %s string, skipping', image_repo, v)
|
logging.debug('image_repo %s is not in %s string, skipping', image_repo, v)
|
||||||
|
@ -315,33 +337,42 @@ def traverse(obj, dict_path=None):
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
"""Small Main program"""
|
"""Small Main program
|
||||||
|
"""
|
||||||
|
|
||||||
parser.add_argument('--in-file', default='versions.yaml',
|
parser.add_argument('--in-file', default='versions.yaml',
|
||||||
help='/path/to/versions.yaml input file; default - "./versions.yaml"')
|
help='/path/to/versions.yaml input file; default - "./versions.yaml"')
|
||||||
|
|
||||||
parser.add_argument('--out-file', default='versions.yaml',
|
parser.add_argument('--out-file', default='versions.yaml',
|
||||||
help='name of output file; default - "versions.yaml" (overwrite existing)')
|
help='name of output file; default - "versions.yaml" (overwrite existing)')
|
||||||
|
|
||||||
parser.add_argument('--skip',
|
parser.add_argument('--skip',
|
||||||
help='comma-delimited list of images and charts to skip during the update')
|
help='comma-delimited list of images and charts to skip during the update')
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
in_file = args.in_file
|
in_file = args.in_file
|
||||||
out_file = args.out_file
|
out_file = args.out_file
|
||||||
if args.skip:
|
if args.skip:
|
||||||
skip_list = tuple(args.skip.strip().split(","))
|
skip_list = tuple(args.skip.strip().split(","))
|
||||||
logging.info('Skip list: %s', skip_list)
|
logging.info("Skip list: %s", skip_list)
|
||||||
else:
|
else:
|
||||||
skip_list = None
|
skip_list = None
|
||||||
|
|
||||||
|
if os.path.basename(out_file) != out_file:
|
||||||
|
logging.error("Name of the output file must not contain path, " +
|
||||||
|
"but only the file name.")
|
||||||
|
print("\n")
|
||||||
|
parser.print_help()
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
if os.path.isfile(in_file):
|
if os.path.isfile(in_file):
|
||||||
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), out_file)
|
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), out_file)
|
||||||
with open(in_file, 'r') as f:
|
with open(in_file, 'r') as f:
|
||||||
f_old = f.read()
|
f_old = f.read()
|
||||||
versions_data_dict = yaml.safe_load(f_old)
|
versions_data_dict = yaml.safe_load(f_old)
|
||||||
else:
|
else:
|
||||||
logging.error("Can\'t find versions.yaml file.\n")
|
logging.error("Can\'t find versions.yaml file.")
|
||||||
|
print("\n")
|
||||||
parser.print_help()
|
parser.print_help()
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
@ -350,9 +381,9 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
with open(out_file, 'w') as f:
|
with open(out_file, 'w') as f:
|
||||||
if os.path.samefile(in_file, out_file):
|
if os.path.samefile(in_file, out_file):
|
||||||
logging.info('Overwriting %s' % in_file)
|
logging.info("Overwriting %s", in_file)
|
||||||
f.write(yaml.safe_dump(versions_data_dict,
|
f.write(yaml.safe_dump(versions_data_dict,
|
||||||
default_flow_style=False,
|
default_flow_style=False,
|
||||||
explicit_end=True, explicit_start=True,
|
explicit_end=True, explicit_start=True,
|
||||||
width=4096))
|
width=4096))
|
||||||
logging.info('New versions.yaml created as %s' % out_file)
|
logging.info("New versions.yaml created as %s", out_file)
|
||||||
|
|
Loading…
Reference in New Issue