[WIP] Add NC converter utility
Change-Id: Ia171353b7720d916e5a99e96922e90fa29d8de26
This commit is contained in:
Roman Gorshunov
parent
ad9a234dd9
commit
517a32bed7
|
|
@ -0,0 +1,295 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
#
|
||||
# versions.yaml file converter tool
|
||||
#
|
||||
# Being run in directory with versions.yaml, will create versions.new.yaml,
|
||||
# without key names and key paths you want to filter out.
|
||||
#
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import datetime
|
||||
from functools import reduce
|
||||
import json
|
||||
import logging
|
||||
import operator
|
||||
import os
|
||||
import requests
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
import git
|
||||
import yaml
|
||||
except ImportError as e:
|
||||
sys.exit(
|
||||
"Failed to import git/yaml libraries needed to run "
|
||||
"this tool %s" % str(e))
|
||||
|
||||
descr_text = (
|
||||
"Being run in directory with versions.yaml, will create versions.new.yaml,"
|
||||
" without key names and key paths you want to filter out."
|
||||
)
|
||||
parser = argparse.ArgumentParser(description=descr_text)
|
||||
|
||||
# Dictionary containing container image repository url to git url mapping
|
||||
#
|
||||
# We expect that each image in container image repository has image tag which
|
||||
# equals to the git commit id of the HEAD in corresponding git repository.
|
||||
#
|
||||
# NOTE(roman_g): currently this is not the case, and image is built/tagged not
|
||||
# on every merge, and there could be a few hours delay between merge and image
|
||||
# re-built and published due to the OpenStack Foundation Zuul infrastructure
|
||||
# being overloaded.
|
||||
image_repo_git_url = {
|
||||
# airflow image is built from airship-shipyard repository
|
||||
"quay.io/airshipit/airflow": "https://opendev.org/airship/shipyard",
|
||||
"quay.io/airshipit/armada": "https://opendev.org/airship/armada",
|
||||
"quay.io/airshipit/deckhand": "https://opendev.org/airship/deckhand",
|
||||
# yes, divingbell image is just Ubuntu 16.04 image,
|
||||
# and we don't check it's tag:
|
||||
#"docker.io/ubuntu": "https://opendev.org/airship/divingbell",
|
||||
"quay.io/airshipit/drydock": "https://opendev.org/airship/drydock",
|
||||
# maas-{rack,region}-controller images are built
|
||||
# from airship-maas repository:
|
||||
"quay.io/airshipit/maas-rack-controller": "https://opendev.org/airship/maas",
|
||||
"quay.io/airshipit/maas-region-controller": "https://opendev.org/airship/maas",
|
||||
"quay.io/airshipit/pegleg": "https://opendev.org/airship/pegleg",
|
||||
"quay.io/airshipit/promenade": "https://opendev.org/airship/promenade",
|
||||
"quay.io/airshipit/shipyard": "https://opendev.org/airship/shipyard",
|
||||
# sstream-cache image is built from airship-maas repository
|
||||
"quay.io/airshipit/sstream-cache": "https://opendev.org/airship/maas"
|
||||
}
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
def __represent_multiline_yaml_str():
|
||||
"""Compel ``yaml`` library to use block style literals for multi-line
|
||||
strings to prevent unwanted multiple newlines.
|
||||
"""
|
||||
|
||||
yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str
|
||||
|
||||
def repr_str(dumper, data):
|
||||
if "\n" in data:
|
||||
return dumper.represent_scalar(
|
||||
"tag:yaml.org,2002:str", data, style="|")
|
||||
return dumper.org_represent_str(data)
|
||||
|
||||
yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
|
||||
|
||||
|
||||
__represent_multiline_yaml_str()
|
||||
|
||||
|
||||
def inverse_dict(dic):
|
||||
"""Accepts dictionary, returns dictionary where keys become values,
|
||||
and values become keys"""
|
||||
new_dict = {}
|
||||
for k, v in dic.items():
|
||||
new_dict[v] = k
|
||||
return new_dict
|
||||
|
||||
|
||||
# https://stackoverflow.com/a/14692747
|
||||
def get_by_path(root, items):
|
||||
"""Access a nested object in root by item sequence."""
|
||||
return reduce(operator.getitem, items, root)
|
||||
|
||||
|
||||
def set_by_path(root, items, value):
|
||||
"""Set a value in a nested object in root by item sequence."""
|
||||
get_by_path(root, items[:-1])[items[-1]] = value
|
||||
|
||||
|
||||
# Based on http://nvie.com/posts/modifying-deeply-nested-structures/
|
||||
def traverse(obj, dict_path=None):
|
||||
"""Accepts Python dictionary with values.yaml contents,
|
||||
updates it with latest git commit id's.
|
||||
"""
|
||||
LOG.debug(
|
||||
"traverse: dict_path: %s, object type: %s, object: obj", dict_path,
|
||||
type(obj))
|
||||
# LOG.debug(
|
||||
# "traverse: dict_path: %s, object type: %s, object: %s", dict_path,
|
||||
# type(obj), obj)
|
||||
|
||||
if dict_path is None:
|
||||
dict_path = []
|
||||
|
||||
if isinstance(obj, dict):
|
||||
# It's a dictionary element
|
||||
# LOG.debug("this object %s is a dictionary", obj)
|
||||
LOG.debug("this object is a dictionary")
|
||||
|
||||
if filter_keys:
|
||||
removed = 0
|
||||
obj_copy = {**obj} # Create a shallow copy
|
||||
for k, v in obj_copy.items():
|
||||
# We are only interested in removing keys in yaml which
|
||||
# have direct values (string, boolean, etc.), not the
|
||||
# keys which have nested structures inside
|
||||
if k in filter_keys and not isinstance(v, dict) and not isinstance(v, list):
|
||||
LOG.info("Deleting key %s from path %s, as it's in a filter keys list", k, dict_path)
|
||||
del obj[k]
|
||||
removed = 1
|
||||
|
||||
if filter_paths_tuples and dict_path != []:
|
||||
for t in filter_paths_tuples:
|
||||
LOG.debug("Current filter path tuple: %s", t)
|
||||
LOG.debug("Last element of current filter path tuple: %s", t[-1])
|
||||
LOG.debug("Current filter path tuple without last element: %s", t[ : -1])
|
||||
# t[-1] - this is the last element of the current filter path
|
||||
# t[ : -1] - this is the current filter path without last element
|
||||
# We check if current path is equal to the filter path w/o last element,
|
||||
# and then if that last element of filter path exists in objent we process,
|
||||
# we remove it.
|
||||
if t[ : -1] == dict_path:
|
||||
if t[-1] in obj:
|
||||
LOG.info("Deleting key %s from path %s, as it's in a filter paths list", t[-1], dict_path)
|
||||
del obj[t[-1]]
|
||||
|
||||
for k, v in obj.items():
|
||||
# If value v we are checking is a dictionary itself, and this
|
||||
# dictionary contains key named "type", and a value of key "type"
|
||||
# equals "git", then
|
||||
# if isinstance(v, dict) and "type" in v and v["type"] == "git":
|
||||
if isinstance(v, dict):
|
||||
# LOG.debug("dict_path: %s", dict_path)
|
||||
LOG.debug("value inside object is a dictionary")
|
||||
|
||||
if "type" in v and v["type"] == "tar":
|
||||
v["location"] = "new_location"
|
||||
v["reference"] = "new_reference"
|
||||
v["subpath"] = "new_subpath"
|
||||
v["type"] = "git"
|
||||
|
||||
else:
|
||||
# LOG.debug("value %s inside object is not a dictionary", v)
|
||||
LOG.debug("value inside object is not a dictionary")
|
||||
|
||||
if isinstance(v, dict) and "type" in v and v["type"] == "tar":
|
||||
v["location"] = "new_location"
|
||||
v["reference"] = "new_reference"
|
||||
v["subpath"] = "new_subpath"
|
||||
v["type"] = "git"
|
||||
|
||||
# Traverse one level deeper
|
||||
traverse(v, dict_path + [k])
|
||||
elif isinstance(obj, list):
|
||||
# It's a list element
|
||||
# LOG.debug("this object %s is a list", obj)
|
||||
LOG.debug("this object is a list")
|
||||
|
||||
for elem in obj:
|
||||
# TODO: Do we have any git references or container image tags in
|
||||
# versions.yaml which are inside lists? Probably not.
|
||||
traverse(elem, dict_path + [[]])
|
||||
else:
|
||||
# It's already a value
|
||||
LOG.debug("this object %s under path %s is a value", obj, dict_path)
|
||||
v = obj
|
||||
|
||||
# Searching for container image repositories, we are only intrested in
|
||||
# strings; there could also be booleans or other types
|
||||
# we are not interested in.
|
||||
if isinstance(v, str):
|
||||
LOG.debug("and %s is a string", v)
|
||||
else:
|
||||
LOG.debug("value %s is not string, skipping", v)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""Main program
|
||||
"""
|
||||
|
||||
parser.add_argument(
|
||||
"--in-file",
|
||||
default="versions-in.yaml",
|
||||
help="/path/to/versions.yaml input file; "
|
||||
"default - \"./versions.yaml\"")
|
||||
parser.add_argument(
|
||||
"--out-file",
|
||||
default="versions-out.yaml",
|
||||
help="name of output file; default - "
|
||||
"\"versions.yaml\" (overwrite existing)")
|
||||
parser.add_argument(
|
||||
"--filter-paths",
|
||||
help="comma-delimited list of yaml paths "
|
||||
"with dot separators to remove from "
|
||||
"resulting yaml output")
|
||||
parser.add_argument(
|
||||
"--filter-keys",
|
||||
help="comma-delimited list of yaml key "
|
||||
"names to remove from resulting yaml output")
|
||||
|
||||
args = parser.parse_args()
|
||||
in_file = args.in_file
|
||||
out_file = args.out_file
|
||||
|
||||
if args.filter_paths:
|
||||
filter_paths = tuple(args.filter_paths.strip().split(","))
|
||||
filter_paths_tuples = []
|
||||
for i in filter_paths:
|
||||
filter_paths_tuples = filter_paths_tuples + [re.split('\.',i)]
|
||||
LOG.info("Filter paths tuples: %s", filter_paths_tuples)
|
||||
else:
|
||||
filter_paths = None
|
||||
|
||||
if args.filter_keys:
|
||||
filter_keys = tuple(args.filter_keys.strip().split(","))
|
||||
LOG.info("Filter keys: %s", filter_keys)
|
||||
else:
|
||||
filter_keys = None
|
||||
|
||||
if os.path.basename(out_file) != out_file:
|
||||
logging.error(
|
||||
"Name of the output file must not contain path, "
|
||||
"but only the file name.")
|
||||
print("\n")
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
if os.path.isfile(in_file):
|
||||
out_file = os.path.join(
|
||||
os.path.dirname(os.path.abspath(in_file)), out_file)
|
||||
with open(in_file, "r") as f:
|
||||
f_old = f.read()
|
||||
versions_data_dict = yaml.safe_load(f_old)
|
||||
else:
|
||||
logging.error("Can't find versions.yaml file.")
|
||||
print("\n")
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
# Traverse loaded yaml and change it
|
||||
traverse(versions_data_dict)
|
||||
|
||||
with open(out_file, "w") as f:
|
||||
if os.path.samefile(in_file, out_file):
|
||||
LOG.info("Overwriting %s", in_file)
|
||||
f.write(
|
||||
yaml.safe_dump(
|
||||
versions_data_dict,
|
||||
default_flow_style=False,
|
||||
explicit_end=True,
|
||||
explicit_start=True,
|
||||
width=4096))
|
||||
LOG.info("New versions.yaml created as %s", out_file)
|
||||
Loading…
Reference in New Issue