Browse Source

versions.yaml updater tool

Being run in directory with versions.yaml or provideded with path to
the versions.yaml, will create, with updated git
commit id's to the latest HEAD in:
  1) references of all charts
  2) tags of container images listed in dictionary image_repo_git_url

Change-Id: I163304da5bf5ec198deb7aaea0abac04c96c569e
Roman Gorshunov 3 years ago
1 changed files with 229 additions and 0 deletions
  1. +229

+ 229
- 0
tools/ View File

@ -0,0 +1,229 @@
#!/usr/bin/env python3
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
# versions.yaml file updater tool
# Being run in directory with versions.yaml, will create,
# with updated git commit id's to the latest HEAD in:
# 1) references of all charts
# 2) tags of container images listed in dict `image_repo_git_url` in the
# code below
from functools import reduce
import argparse
import logging
import operator
import os
import sys
import git
import yaml
except ImportError as e:
sys.exit("Failed to import git/yaml libraries needed to run" +
"this tool %s" % str(e))
descr_text="Process versions.yaml and create with \
updated git commit id\'s to the latest HEAD in: \
1) references of all charts \
2) tags of container images listed in dict \
`image_repo_git_url` in the code."
parser = argparse.ArgumentParser(description=descr_text)
# Dictionary containing container image repository url to git url mapping
# We expect that each image in container image repository has image tag which
# equals to the git commit id of the HEAD in corresponding git repository
image_repo_git_url = {
# airflow image is built from airship-shipyard repository
'': '',
'': '',
'': '',
'': '',
'': '',
# maas-{rack,region}-controller images are built from airship-maas repository
'': '',
'': '',
'': '',
'': '',
'': '',
# sstream-cache image is built from airship-maas repository
'': '',
'': ''
# Disabled by Kaspars:
#'': ''
# Temporary dict of git url's and cached commit id's: {'git_url': 'commit_id'}
git_url_commit_ids = {}
dict_path = None
def lsremote(url, remote_ref):
"""Accepts git url and remote reference, returns git commit id."""
git_commit_id_remote_ref = {}
g = git.cmd.Git()'Fetching ' + url + ' ' + remote_ref + ' reference...')
hash_ref_list = g.ls_remote(url, remote_ref).split('\t')
git_commit_id_remote_ref[hash_ref_list[1]] = hash_ref_list[0]
return git_commit_id_remote_ref[remote_ref]
def get_commit_id(url):
"""Accepts url of git repo and returns corresponding git commit hash"""
# If we don't have this git url in our url's dictionary,
# fetch latest commit ID and add new dictionary entry
logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
if url not in git_url_commit_ids:
logging.debug('git url: ' + url +
' is not in git_url_commit_ids dict;' +
' adding it with HEAD commit id')
git_url_commit_ids[url] = lsremote(url, 'HEAD')
return git_url_commit_ids[url]
def get_by_path(root, items):
"""Access a nested object in root by item sequence."""
return reduce(operator.getitem, items, root)
def set_by_path(root, items, value):
"""Set a value in a nested object in root by item sequence."""
get_by_path(root, items[:-1])[items[-1]] = value
# Based on
def traverse(obj, dict_path=None):
"""Accepts Python dictionary with values.yaml contents,
updates it with latest git commit id's.
logging.debug('traverse: dict_path: %s, object type: %s, object: %s',
dict_path, type(obj), obj)
if dict_path is None:
dict_path = []
if isinstance(obj, dict):
# It's a dictionary element
logging.debug('this object is a dictionary')
for k, v in obj.items():
# If value v we are checking is a dictionary itself, and this
# dictionary contains key named 'type', and a value of key 'type'
# equals 'git', then
if isinstance(v, dict) and 'type' in v and v['type'] == 'git':
old_git_commit_id = v['reference']
git_url = v['location']
new_git_commit_id = get_commit_id(git_url)
# Update git commit id in reference field of dictionary
if old_git_commit_id != new_git_commit_id:'Updating git reference for chart %s from %s to ' +
'%s (%s)',
k, old_git_commit_id, new_git_commit_id,
v['reference'] = new_git_commit_id
else:'Git reference %s for chart %s is already up to date (%s) ',
old_git_commit_id, k, git_url)
logging.debug('value %s inside object is not a dictionary, or it does not ' +
'contain key \'type\' with value \'git\', skipping', v)
# Traverse one level deeper
traverse(v, dict_path + [k])
elif isinstance(obj, list):
# It's a list element
logging.debug('this object is a list')
for elem in obj:
# TODO: Do we have any git references or container image tags in
# versions.yaml which are inside lists? Probably not.
traverse(elem, dict_path + [[]])
# It's already a value
logging.debug('this object is a value')
v = obj
# Searching for container image repositories, we are only intrested in
# strings; there could also be booleans or other types we are not interested in.
if isinstance(v, str):
for image_repo, git_url in image_repo_git_url.items():
if image_repo in v:
logging.debug('image_repo %s is in %s string', image_repo, v)
# hash_v: {'&whatever repo_url', 'git commit id tag'}
# Note: 'image' below could contain not just image, but also
# '&ref host.domain/path/image'
hash_v = v.split(":")
image, old_git_commit_id = hash_v
new_git_commit_id = get_commit_id(git_url)
# Update git commit id in tag of container image
if old_git_commit_id != new_git_commit_id:'Updating git commit id in' +
' tag of container image %s from %s to' +
' %s (%s)',
image, old_git_commit_id,
new_git_commit_id, git_url)
set_by_path(versions_data_dict, dict_path, image + ':' + new_git_commit_id)
else:'Git tag %s for container ' +
'image %s is already up to date (%s)',
old_git_commit_id, image, git_url)
logging.debug('image_repo %s is not in %s string, skipping', image_repo, v)
logging.debug('value %s is not string, skipping', v)
if __name__ == '__main__':
"""Small Main program"""
parser.add_argument('--in-file', default = 'versions.yaml', help = '/path/to/versions.yaml file')
args = parser.parse_args()
in_file = args.in_file
if os.path.isfile(in_file):
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), '')
with open(in_file, 'r') as f:
f_old =
versions_data_dict = yaml.safe_load(f_old)
logging.error("Can\'t find versions.yaml file.\n")
# Traverse loaded yaml and change it
with open(out_file, 'w') as f:
default_flow_style=False, default_style='\"',
explicit_end=True, explicit_start=True,
width=4096))'New versions.yaml created as %s' % out_file)