airship
/
treasuremap
Code Issues Proposed changes
Browse Source

Standardize Treasuremap code with YAPF 

From recently merged document updates in [0] there is a desire to
standardize the Airship project python codebase.  This is the effort
to do so for the Treasuremap project.

[0] https://review.opendev.org/#/c/671291/

Change-Id: Icd45f1b99a90e6c934a84fdd91f2f7f8af5a8ddb
tags/v1.3
HUGHES, ALEXANDER (ah8742) 4 months ago
parent
8a94b9bf58
commit
8d0b847a03
6 changed files with 171 additions and 95 deletions
Split View Show Diff Stats
  1. +2
    -0
      .gitignore
  2. +10
    -0
      .style.yapf
  3. +5
    -0
      Makefile
  4. +3
    -0
      test-requirements.txt
  5. +117
    -94
      tools/updater.py
  6. +34
    -1
      tox.ini

+ 2
- 0
.gitignore View File

@@ -0,0 +1,2 @@
# Unit test / coverage reports
.tox/

+ 10
- 0
.style.yapf View File

@@ -0,0 +1,10 @@
[style]
based_on_style = pep8
spaces_before_comment = 2
column_limit = 79
blank_line_before_nested_class_or_def = false
blank_line_before_module_docstring = true
split_before_logical_operator = true
split_before_first_argument = true
allow_split_before_dict_value = false
split_before_arithmetic_operator = true

+ 5
- 0
Makefile View File

@@ -25,3 +25,8 @@ docs: clean build_docs
.PHONY: build_docs
build_docs:
tox -e docs

# Perform auto formatting
.PHONY: format
format:
tox -e fmt

+ 3
- 0
test-requirements.txt View File

@@ -0,0 +1,3 @@
yapf==0.27.0
flake8-import-order==0.18.1
bandit==1.6.0

+ 117
- 94
tools/updater.py View File

@@ -42,15 +42,17 @@ try:
import git
import yaml
except ImportError as e:
sys.exit("Failed to import git/yaml libraries needed to run "
"this tool %s" % str(e))

descr_text = ("Being run in directory with versions.yaml, will create "
"versions.new.yaml, with updated git commit id's to the "
"latest HEAD in references of all charts. In addition to "
"that, the tool updates references to the container images "
"with the tag, equal to the latest image which exists on "
"quay.io repository and is available for download.")
sys.exit(
"Failed to import git/yaml libraries needed to run "
"this tool %s" % str(e))

descr_text = (
"Being run in directory with versions.yaml, will create "
"versions.new.yaml, with updated git commit id's to the "
"latest HEAD in references of all charts. In addition to "
"that, the tool updates references to the container images "
"with the tag, equal to the latest image which exists on "
"quay.io repository and is available for download.")
parser = argparse.ArgumentParser(description=descr_text)

# Dictionary containing container image repository url to git url mapping
@@ -73,10 +75,8 @@ image_repo_git_url = {
"quay.io/airshipit/drydock": "https://opendev.org/airship/drydock",
# maas-{rack,region}-controller images are built
# from airship-maas repository:
"quay.io/airshipit/maas-rack-controller":
"https://opendev.org/airship/maas",
"quay.io/airshipit/maas-region-controller":
"https://opendev.org/airship/maas",
"quay.io/airshipit/maas-rack-controller": "https://opendev.org/airship/maas",
"quay.io/airshipit/maas-region-controller": "https://opendev.org/airship/maas",
"quay.io/airshipit/pegleg": "https://opendev.org/airship/pegleg",
"quay.io/airshipit/promenade": "https://opendev.org/airship/promenade",
"quay.io/airshipit/shipyard": "https://opendev.org/airship/shipyard",
@@ -147,9 +147,9 @@ def get_commit_id(url):
# fetch latest commit ID and add new dictionary entry
LOG.debug("git_url_commit_ids: %s", git_url_commit_ids)
if url not in git_url_commit_ids:
LOG.debug("git url: %s " +
"is not in git_url_commit_ids dict; "
"adding it with HEAD commit id", url)
LOG.debug(
"git url: %s " + "is not in git_url_commit_ids dict; "
"adding it with HEAD commit id", url)
git_url_commit_ids[url] = lsremote(url, "HEAD")

return git_url_commit_ids[url]
@@ -160,18 +160,19 @@ def get_image_tag(image):
returns 0 (image not hosted on quay.io), True, or False
"""
if not image.startswith("quay.io/"):
LOG.info("Unable to verify if image %s "
"is in containers repository: only quay.io is "
"supported at the moment", image)
LOG.info(
"Unable to verify if image %s "
"is in containers repository: only quay.io is "
"supported at the moment", image)
return 0

# If we don't have this image in our images's dictionary,
# fetch latest tag and add new dictionary entry
LOG.debug("image_repo_tags: %s", image_repo_tags)
if image not in image_repo_tags:
LOG.debug("image: %s " +
"is not in image_repo_tags dict; "
"adding it with latest tag", image)
LOG.debug(
"image: %s " + "is not in image_repo_tags dict; "
"adding it with latest tag", image)
image_repo_tags[image] = get_image_latest_tag(image)

return image_repo_tags[image]
@@ -198,8 +199,9 @@ def get_image_latest_tag(image):
if res.ok:
break
except requests.exceptions.Timeout:
LOG.warning("Failed to fetch url %s for %d/%d attempt(s)",
url, attempt, max_attempts)
LOG.warning(
"Failed to fetch url %s for %d/%d attempt(s)", url, attempt,
max_attempts)
time.sleep(5)
except requests.exceptions.TooManyRedirects:
logging.error("Failed to fetch url %s, TooManyRedirects", url)
@@ -208,18 +210,19 @@ def get_image_latest_tag(image):
logging.error("Failed to fetch url %s, error: %s", url, e)
return 0
if attempt == max_attempts:
logging.error("Failed to connect to quay.io for %d attempt(s)",
attempt)
logging.error(
"Failed to connect to quay.io for %d attempt(s)", attempt)
return 0

if res.status_code != 200:
logging.error("Image %s is not available on quay.io or "
"requires authentication", image)
logging.error(
"Image %s is not available on quay.io or "
"requires authentication", image)
return 0

try:
res = res.json()
except json.decoder.JSONDecodeError: # pylint: disable=no-member
except json.decoder.JSONDecodeError: # pylint: disable=no-member
logging.error("Unable to parse response from quay.io (%s)", res.url)
return 0

@@ -240,8 +243,9 @@ def get_image_latest_tag(image):
if tag_filter in tag["name"]:
return tag["name"]

LOG.info("Skipping tag %s as not matching to the filter %s",
tag["name"], tag_filter)
LOG.info(
"Skipping tag %s as not matching to the filter %s",
tag["name"], tag_filter)
if not possible_tag:
possible_tag = tag["name"]

@@ -273,8 +277,9 @@ def traverse(obj, dict_path=None):
"""Accepts Python dictionary with values.yaml contents,
updates it with latest git commit id's.
"""
LOG.debug("traverse: dict_path: %s, object type: %s, object: %s",
dict_path, type(obj), obj)
LOG.debug(
"traverse: dict_path: %s, object type: %s, object: %s", dict_path,
type(obj), obj)

if dict_path is None:
dict_path = []
@@ -293,27 +298,29 @@ def traverse(obj, dict_path=None):
git_url = v["location"]

if skip_list and k in skip_list:
LOG.info("Ignoring chart %s, it is in a "
"skip list (%s)", k, git_url)
LOG.info(
"Ignoring chart %s, it is in a "
"skip list (%s)", k, git_url)
continue

new_git_commit_id = get_commit_id(git_url)

# Update git commit id in reference field of dictionary
if old_git_commit_id != new_git_commit_id:
LOG.info("Updating git reference for "
"chart %s from %s to %s (%s)",
k, old_git_commit_id, new_git_commit_id,
git_url)
LOG.info(
"Updating git reference for "
"chart %s from %s to %s (%s)", k, old_git_commit_id,
new_git_commit_id, git_url)
v["reference"] = new_git_commit_id
else:
LOG.info("Git reference %s for chart %s is already "
"up to date (%s)",
old_git_commit_id, k, git_url)
LOG.info(
"Git reference %s for chart %s is already "
"up to date (%s)", old_git_commit_id, k, git_url)
else:
LOG.debug("value %s inside object is not a dictionary, "
"or it does not contain key \"type\" with "
"value \"git\", skipping", v)
LOG.debug(
"value %s inside object is not a dictionary, "
"or it does not contain key \"type\" with "
"value \"git\", skipping", v)

# Traverse one level deeper
traverse(v, dict_path + [k])
@@ -336,8 +343,7 @@ def traverse(obj, dict_path=None):
if isinstance(v, str):
for image_repo in image_repo_git_url:
if image_repo in v:
LOG.debug("image_repo %s is in %s string",
image_repo, v)
LOG.debug("image_repo %s is in %s string", image_repo, v)

# hash_v: {"&whatever repo_url", "git commit id tag"}
# Note: "image" below could contain not just image,
@@ -346,8 +352,9 @@ def traverse(obj, dict_path=None):
image, old_image_tag = hash_v

if skip_list and image in skip_list:
LOG.info("Ignoring image %s, it is in a "
"skip list", image)
LOG.info(
"Ignoring image %s, it is in a "
"skip list", image)
continue

new_image_tag = get_image_tag(image)
@@ -357,19 +364,23 @@ def traverse(obj, dict_path=None):

# Update git commit id in tag of container image
if old_image_tag != new_image_tag:
LOG.info("Updating git commit id in "
"tag of container image %s from %s to %s",
image, old_image_tag, new_image_tag)
set_by_path(versions_data_dict, dict_path,
image + ":" + new_image_tag)
LOG.info(
"Updating git commit id in "
"tag of container image %s from %s to %s", image,
old_image_tag, new_image_tag)
set_by_path(
versions_data_dict, dict_path,
image + ":" + new_image_tag)

else:
LOG.info("Git tag %s for container "
"image %s is already up to date",
old_image_tag, image)
LOG.info(
"Git tag %s for container "
"image %s is already up to date", old_image_tag,
image)
else:
LOG.debug("image_repo %s is not in %s string, "
"skipping", image_repo, v)
LOG.debug(
"image_repo %s is not in %s string, "
"skipping", image_repo, v)
else:
LOG.debug("value %s is not string, skipping", v)

@@ -387,8 +398,8 @@ def print_versions_table():

table_format = "{:48s} {:60s} {:54s} {:41s}\n"
table_content = "\n"
table_content += table_format.format("Image repo","Git repo",
"Image repo tag","Git repo Commit ID")
table_content += table_format.format(
"Image repo", "Git repo", "Image repo tag", "Git repo Commit ID")

# Copy dicts for later modification
image_repo_tags_copy = copy.deepcopy(image_repo_tags)
@@ -408,10 +419,9 @@ def print_versions_table():
if not git_repo in git_url_commit_ids_copy:
git_url_commit_ids_copy[git_repo] = lsremote(git_repo, "HEAD")

table_content += table_format.format(image_repo,
git_repo,
image_repo_tags_copy[image_repo],
git_url_commit_ids_copy[git_repo])
table_content += table_format.format(
image_repo, git_repo, image_repo_tags_copy[image_repo],
git_url_commit_ids_copy[git_repo])

LOG.info("")
for line in table_content.splitlines():
@@ -447,12 +457,14 @@ def print_missing_references():
for ref in missing_references:
LOG.warning(missing_references[ref])
LOG.warning("")
LOG.warning("Refs which are not in git_url_commit_ids mean that "
"we have not been updating chart references (or "
"there are no charts referred in versions.yaml)")
LOG.warning("Refs which are not in image_repo_tags mean that we "
"have not been updating image tags (or there are no "
"images referred in versions.yaml)")
LOG.warning(
"Refs which are not in git_url_commit_ids mean that "
"we have not been updating chart references (or "
"there are no charts referred in versions.yaml)")
LOG.warning(
"Refs which are not in image_repo_tags mean that we "
"have not been updating image tags (or there are no "
"images referred in versions.yaml)")
LOG.warning("")


@@ -483,7 +495,8 @@ def print_outdated_images():
# This is where we check if there is tag matching commit_id exists,
# and if not, then we append that image_repo to the list of
# possibly outdated images
if git_url_commit_ids_copy[git_repo] not in image_repo_tags_copy[image_repo]:
if git_url_commit_ids_copy[git_repo] not in image_repo_tags_copy[
image_repo]:
possibly_outdated_images.append(image_repo)

if possibly_outdated_images:
@@ -498,20 +511,26 @@ if __name__ == "__main__":
"""Main program
"""

parser.add_argument("--in-file", default="versions.yaml",
help="/path/to/versions.yaml input file; "
"default - \"./versions.yaml\"")
parser.add_argument("--out-file", default="versions.yaml",
help="name of output file; default - "
"\"versions.yaml\" (overwrite existing)")
parser.add_argument("--skip",
help="comma-delimited list of images and charts "
"to skip during the update; e.g. \"ceph\" "
"will skip all charts and images which have "
"\"ceph\" in the name")
parser.add_argument('--tag-filter',
help="e.g. \"ubuntu\"; update would use image ref. "
"tags on quay.io matching the filter")
parser.add_argument(
"--in-file",
default="versions.yaml",
help="/path/to/versions.yaml input file; "
"default - \"./versions.yaml\"")
parser.add_argument(
"--out-file",
default="versions.yaml",
help="name of output file; default - "
"\"versions.yaml\" (overwrite existing)")
parser.add_argument(
"--skip",
help="comma-delimited list of images and charts "
"to skip during the update; e.g. \"ceph\" "
"will skip all charts and images which have "
"\"ceph\" in the name")
parser.add_argument(
'--tag-filter',
help="e.g. \"ubuntu\"; update would use image ref. "
"tags on quay.io matching the filter")

args = parser.parse_args()
in_file = args.in_file
@@ -526,15 +545,16 @@ if __name__ == "__main__":
LOG.info("Tag filter: %s", tag_filter)

if os.path.basename(out_file) != out_file:
logging.error("Name of the output file must not contain path, "
"but only the file name.")
logging.error(
"Name of the output file must not contain path, "
"but only the file name.")
print("\n")
parser.print_help()
sys.exit(1)

if os.path.isfile(in_file):
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)),
out_file)
out_file = os.path.join(
os.path.dirname(os.path.abspath(in_file)), out_file)
with open(in_file, "r") as f:
f_old = f.read()
versions_data_dict = yaml.safe_load(f_old)
@@ -554,8 +574,11 @@ if __name__ == "__main__":
with open(out_file, "w") as f:
if os.path.samefile(in_file, out_file):
LOG.info("Overwriting %s", in_file)
f.write(yaml.safe_dump(versions_data_dict,
default_flow_style=False,
explicit_end=True, explicit_start=True,
width=4096))
f.write(
yaml.safe_dump(
versions_data_dict,
default_flow_style=False,
explicit_end=True,
explicit_start=True,
width=4096))
LOG.info("New versions.yaml created as %s", out_file)

+ 34
- 1
tox.ini View File

@@ -1,6 +1,8 @@
[tox]
# Allows docs to be built without setup.py having to exist. Requires that
# usedevelop be False as well (which it is by default).
envlist = pep8
minversion = 2.3.1
skipsdist = True

[testenv]
@@ -15,7 +17,38 @@ commands = {posargs}
[testenv:docs]
basepython = python3
whitelist_externals = rm
deps = -r{toxinidir}/doc/requirements.txt
deps =
-r{toxinidir}/doc/requirements.txt
commands =
rm -rf doc/build
sphinx-build -W -b html doc/source doc/build/html

[testenv:fmt]
basepython = python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
yapf -ir {toxinidir}/tools

[testenv:pep8]
basepython = python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
bandit -r {toxinidir}/tools -n 5
flake8 {toxinidir}/tools
yapf -dr {toxinidir}/tools

[flake8]
filename = *.py
show-source = true
# [H106] Don't put vim configuration in source files.
# [H201] No 'except:' at least use 'except Exception:'
# [H904] Delay string interpolations at logging calls.
enable-extensions = H106,H201,H904
# [W503] line break before binary operator
ignore = W503
exclude=.venv,.git,.tox,build,dist,*lib/python*,*egg,tools,*.ini,*.po,*.pot
max-complexity = 24
application-import-names = treasuremap
import-order-style = pep8

Write Preview
Loading…
Cancel
Save