Browse Source
Being run in directory with versions.yaml or provideded with path to the versions.yaml, will create versions.new.yaml, with updated git commit id's to the latest HEAD in: 1) references of all charts 2) tags of container images listed in dictionary image_repo_git_url dict Change-Id: I163304da5bf5ec198deb7aaea0abac04c96c569echanges/09/596909/23
1 changed files with 229 additions and 0 deletions
@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3 |
||||
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# |
||||
# versions.yaml file updater tool |
||||
# |
||||
# Being run in directory with versions.yaml, will create versions.new.yaml, |
||||
# with updated git commit id's to the latest HEAD in: |
||||
# 1) references of all charts |
||||
# 2) tags of container images listed in dict `image_repo_git_url` in the |
||||
# code below |
||||
# |
||||
|
||||
from functools import reduce |
||||
import argparse |
||||
import logging |
||||
import operator |
||||
import os |
||||
import sys |
||||
|
||||
try: |
||||
import git |
||||
import yaml |
||||
except ImportError as e: |
||||
sys.exit("Failed to import git/yaml libraries needed to run" + |
||||
"this tool %s" % str(e)) |
||||
|
||||
descr_text="Process versions.yaml and create versions.new.yaml with \ |
||||
updated git commit id\'s to the latest HEAD in: \ |
||||
1) references of all charts \ |
||||
2) tags of container images listed in dict \ |
||||
`image_repo_git_url` in the code." |
||||
parser = argparse.ArgumentParser(description=descr_text) |
||||
|
||||
# Dictionary containing container image repository url to git url mapping |
||||
# |
||||
# We expect that each image in container image repository has image tag which |
||||
# equals to the git commit id of the HEAD in corresponding git repository |
||||
image_repo_git_url = { |
||||
# airflow image is built from airship-shipyard repository |
||||
'quay.io/airshipit/airflow': 'https://git.openstack.org/openstack/airship-shipyard', |
||||
'quay.io/airshipit/armada': 'https://git.openstack.org/openstack/airship-armada', |
||||
'quay.io/airshipit/deckhand': 'https://git.openstack.org/openstack/airship-deckhand', |
||||
'quay.io/airshipit/divingbell': 'https://git.openstack.org/openstack/airship-divingbell', |
||||
'quay.io/airshipit/drydock': 'https://git.openstack.org/openstack/airship-drydock', |
||||
# maas-{rack,region}-controller images are built from airship-maas repository |
||||
'quay.io/airshipit/maas-rack-controller': 'https://git.openstack.org/openstack/airship-maas', |
||||
'quay.io/airshipit/maas-region-controller': 'https://git.openstack.org/openstack/airship-maas', |
||||
'quay.io/airshipit/pegleg': 'https://git.openstack.org/openstack/airship-pegleg', |
||||
'quay.io/airshipit/promenade': 'https://git.openstack.org/openstack/airship-promenade', |
||||
'quay.io/airshipit/shipyard': 'https://git.openstack.org/openstack/airship-shipyard', |
||||
# sstream-cache image is built from airship-maas repository |
||||
'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas', |
||||
'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios' |
||||
# Disabled by Kaspars: https://review.openstack.org/#/c/596909/21/tools/updater.py@53 |
||||
#'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter' |
||||
} |
||||
|
||||
logging.basicConfig(level=logging.INFO) |
||||
|
||||
# Temporary dict of git url's and cached commit id's: {'git_url': 'commit_id'} |
||||
git_url_commit_ids = {} |
||||
dict_path = None |
||||
|
||||
|
||||
# https://stackoverflow.com/a/35585837 |
||||
def lsremote(url, remote_ref): |
||||
"""Accepts git url and remote reference, returns git commit id.""" |
||||
git_commit_id_remote_ref = {} |
||||
g = git.cmd.Git() |
||||
logging.info('Fetching ' + url + ' ' + remote_ref + ' reference...') |
||||
hash_ref_list = g.ls_remote(url, remote_ref).split('\t') |
||||
git_commit_id_remote_ref[hash_ref_list[1]] = hash_ref_list[0] |
||||
return git_commit_id_remote_ref[remote_ref] |
||||
|
||||
|
||||
def get_commit_id(url): |
||||
"""Accepts url of git repo and returns corresponding git commit hash""" |
||||
# If we don't have this git url in our url's dictionary, |
||||
# fetch latest commit ID and add new dictionary entry |
||||
logging.debug('git_url_commit_ids: %s', git_url_commit_ids) |
||||
if url not in git_url_commit_ids: |
||||
logging.debug('git url: ' + url + |
||||
' is not in git_url_commit_ids dict;' + |
||||
' adding it with HEAD commit id') |
||||
git_url_commit_ids[url] = lsremote(url, 'HEAD') |
||||
return git_url_commit_ids[url] |
||||
|
||||
|
||||
# https://stackoverflow.com/a/14692747 |
||||
def get_by_path(root, items): |
||||
"""Access a nested object in root by item sequence.""" |
||||
return reduce(operator.getitem, items, root) |
||||
|
||||
|
||||
def set_by_path(root, items, value): |
||||
"""Set a value in a nested object in root by item sequence.""" |
||||
get_by_path(root, items[:-1])[items[-1]] = value |
||||
|
||||
|
||||
# Based on http://nvie.com/posts/modifying-deeply-nested-structures/ |
||||
def traverse(obj, dict_path=None): |
||||
"""Accepts Python dictionary with values.yaml contents, |
||||
updates it with latest git commit id's. |
||||
""" |
||||
logging.debug('traverse: dict_path: %s, object type: %s, object: %s', |
||||
dict_path, type(obj), obj) |
||||
|
||||
if dict_path is None: |
||||
dict_path = [] |
||||
|
||||
if isinstance(obj, dict): |
||||
# It's a dictionary element |
||||
logging.debug('this object is a dictionary') |
||||
|
||||
for k, v in obj.items(): |
||||
# If value v we are checking is a dictionary itself, and this |
||||
# dictionary contains key named 'type', and a value of key 'type' |
||||
# equals 'git', then |
||||
if isinstance(v, dict) and 'type' in v and v['type'] == 'git': |
||||
|
||||
old_git_commit_id = v['reference'] |
||||
git_url = v['location'] |
||||
|
||||
new_git_commit_id = get_commit_id(git_url) |
||||
|
||||
# Update git commit id in reference field of dictionary |
||||
if old_git_commit_id != new_git_commit_id: |
||||
logging.info('Updating git reference for chart %s from %s to ' + |
||||
'%s (%s)', |
||||
k, old_git_commit_id, new_git_commit_id, |
||||
git_url) |
||||
v['reference'] = new_git_commit_id |
||||
else: |
||||
logging.info('Git reference %s for chart %s is already up to date (%s) ', |
||||
old_git_commit_id, k, git_url) |
||||
else: |
||||
logging.debug('value %s inside object is not a dictionary, or it does not ' + |
||||
'contain key \'type\' with value \'git\', skipping', v) |
||||
|
||||
# Traverse one level deeper |
||||
traverse(v, dict_path + [k]) |
||||
elif isinstance(obj, list): |
||||
# It's a list element |
||||
logging.debug('this object is a list') |
||||
|
||||
for elem in obj: |
||||
# TODO: Do we have any git references or container image tags in |
||||
# versions.yaml which are inside lists? Probably not. |
||||
traverse(elem, dict_path + [[]]) |
||||
else: |
||||
# It's already a value |
||||
logging.debug('this object is a value') |
||||
v = obj |
||||
|
||||
# Searching for container image repositories, we are only intrested in |
||||
# strings; there could also be booleans or other types we are not interested in. |
||||
if isinstance(v, str): |
||||
for image_repo, git_url in image_repo_git_url.items(): |
||||
if image_repo in v: |
||||
logging.debug('image_repo %s is in %s string', image_repo, v) |
||||
|
||||
# hash_v: {'&whatever repo_url', 'git commit id tag'} |
||||
# Note: 'image' below could contain not just image, but also |
||||
# '&ref host.domain/path/image' |
||||
hash_v = v.split(":") |
||||
image, old_git_commit_id = hash_v |
||||
|
||||
new_git_commit_id = get_commit_id(git_url) |
||||
|
||||
# Update git commit id in tag of container image |
||||
if old_git_commit_id != new_git_commit_id: |
||||
logging.info('Updating git commit id in' + |
||||
' tag of container image %s from %s to' + |
||||
' %s (%s)', |
||||
image, old_git_commit_id, |
||||
new_git_commit_id, git_url) |
||||
set_by_path(versions_data_dict, dict_path, image + ':' + new_git_commit_id) |
||||
|
||||
else: |
||||
logging.info('Git tag %s for container ' + |
||||
'image %s is already up to date (%s)', |
||||
old_git_commit_id, image, git_url) |
||||
else: |
||||
logging.debug('image_repo %s is not in %s string, skipping', image_repo, v) |
||||
else: |
||||
logging.debug('value %s is not string, skipping', v) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
"""Small Main program""" |
||||
|
||||
parser.add_argument('--in-file', default = 'versions.yaml', help = '/path/to/versions.yaml file') |
||||
args = parser.parse_args() |
||||
|
||||
in_file = args.in_file |
||||
|
||||
if os.path.isfile(in_file): |
||||
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), 'versions.new.yaml') |
||||
with open(in_file, 'r') as f: |
||||
f_old = f.read() |
||||
versions_data_dict = yaml.safe_load(f_old) |
||||
else: |
||||
logging.error("Can\'t find versions.yaml file.\n") |
||||
parser.print_help() |
||||
sys.exit(1) |
||||
|
||||
# Traverse loaded yaml and change it |
||||
traverse(versions_data_dict) |
||||
|
||||
with open(out_file, 'w') as f: |
||||
f.write(yaml.safe_dump(versions_data_dict, |
||||
default_flow_style=False, default_style='\"', |
||||
explicit_end=True, explicit_start=True, |
||||
width=4096)) |
||||
logging.info('New versions.yaml created as %s' % out_file) |
Loading…
Reference in new issue