airship/treasuremap
Code Issues Proposed changes

Standardize Treasuremap code with YAPF

Browse Source 
From recently merged document updates in [0] there is a desire to
standardize the Airship project python codebase.  This is the effort
to do so for the Treasuremap project.

[0] https://review.opendev.org/#/c/671291/

Change-Id: Icd45f1b99a90e6c934a84fdd91f2f7f8af5a8ddb
This commit is contained in:
HUGHES, ALEXANDER (ah8742) 2019-07-24 08:38:34 -05:00 committed by Alexander Hughes
parent 8a94b9bf58
commit 8d0b847a03
6 changed files with 170 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# Unit test / coverage reports
.tox/

10
.style.yapf Normal file
View File

@ -0,0 +1,10 @@
[style]
based_on_style = pep8
spaces_before_comment = 2
column_limit = 79
blank_line_before_nested_class_or_def = false
blank_line_before_module_docstring = true
split_before_logical_operator = true
split_before_first_argument = true
allow_split_before_dict_value = false
split_before_arithmetic_operator = true

5
Makefile
View File

@ -25,3 +25,8 @@ docs: clean build_docs
.PHONY: build_docs .PHONY: build_docs
build_docs: build_docs:
tox -e docs tox -e docs
# Perform auto formatting
.PHONY: format
format:
tox -e fmt

3
test-requirements.txt Normal file
View File

@ -0,0 +1,3 @@
yapf==0.27.0
flake8-import-order==0.18.1
bandit==1.6.0

209
tools/updater.py
View File

@ -42,15 +42,17 @@ try:
import git import git
import yaml import yaml
except ImportError as e: except ImportError as e:
sys.exit("Failed to import git/yaml libraries needed to run " sys.exit(
"this tool %s" % str(e)) "Failed to import git/yaml libraries needed to run "
"this tool %s" % str(e))
descr_text = ("Being run in directory with versions.yaml, will create " descr_text = (
"versions.new.yaml, with updated git commit id's to the " "Being run in directory with versions.yaml, will create "
"latest HEAD in references of all charts. In addition to " "versions.new.yaml, with updated git commit id's to the "
"that, the tool updates references to the container images " "latest HEAD in references of all charts. In addition to "
"with the tag, equal to the latest image which exists on " "that, the tool updates references to the container images "
"quay.io repository and is available for download.") "with the tag, equal to the latest image which exists on "
"quay.io repository and is available for download.")
parser = argparse.ArgumentParser(description=descr_text) parser = argparse.ArgumentParser(description=descr_text)
# Dictionary containing container image repository url to git url mapping # Dictionary containing container image repository url to git url mapping
@ -73,10 +75,8 @@ image_repo_git_url = {
"quay.io/airshipit/drydock": "https://opendev.org/airship/drydock", "quay.io/airshipit/drydock": "https://opendev.org/airship/drydock",
# maas-{rack,region}-controller images are built # maas-{rack,region}-controller images are built
# from airship-maas repository: # from airship-maas repository:
"quay.io/airshipit/maas-rack-controller": "quay.io/airshipit/maas-rack-controller": "https://opendev.org/airship/maas",
"https://opendev.org/airship/maas", "quay.io/airshipit/maas-region-controller": "https://opendev.org/airship/maas",
"quay.io/airshipit/maas-region-controller":
"https://opendev.org/airship/maas",
"quay.io/airshipit/pegleg": "https://opendev.org/airship/pegleg", "quay.io/airshipit/pegleg": "https://opendev.org/airship/pegleg",
"quay.io/airshipit/promenade": "https://opendev.org/airship/promenade", "quay.io/airshipit/promenade": "https://opendev.org/airship/promenade",
"quay.io/airshipit/shipyard": "https://opendev.org/airship/shipyard", "quay.io/airshipit/shipyard": "https://opendev.org/airship/shipyard",
@ -147,9 +147,9 @@ def get_commit_id(url):
# fetch latest commit ID and add new dictionary entry # fetch latest commit ID and add new dictionary entry
LOG.debug("git_url_commit_ids: %s", git_url_commit_ids) LOG.debug("git_url_commit_ids: %s", git_url_commit_ids)
if url not in git_url_commit_ids: if url not in git_url_commit_ids:
LOG.debug("git url: %s " + LOG.debug(
"is not in git_url_commit_ids dict; " "git url: %s " + "is not in git_url_commit_ids dict; "
"adding it with HEAD commit id", url) "adding it with HEAD commit id", url)
git_url_commit_ids[url] = lsremote(url, "HEAD") git_url_commit_ids[url] = lsremote(url, "HEAD")
return git_url_commit_ids[url] return git_url_commit_ids[url]
@ -160,18 +160,19 @@ def get_image_tag(image):
returns 0 (image not hosted on quay.io), True, or False returns 0 (image not hosted on quay.io), True, or False
""" """
if not image.startswith("quay.io/"): if not image.startswith("quay.io/"):
LOG.info("Unable to verify if image %s " LOG.info(
"is in containers repository: only quay.io is " "Unable to verify if image %s "
"supported at the moment", image) "is in containers repository: only quay.io is "
"supported at the moment", image)
return 0 return 0
# If we don't have this image in our images's dictionary, # If we don't have this image in our images's dictionary,
# fetch latest tag and add new dictionary entry # fetch latest tag and add new dictionary entry
LOG.debug("image_repo_tags: %s", image_repo_tags) LOG.debug("image_repo_tags: %s", image_repo_tags)
if image not in image_repo_tags: if image not in image_repo_tags:
LOG.debug("image: %s " + LOG.debug(
"is not in image_repo_tags dict; " "image: %s " + "is not in image_repo_tags dict; "
"adding it with latest tag", image) "adding it with latest tag", image)
image_repo_tags[image] = get_image_latest_tag(image) image_repo_tags[image] = get_image_latest_tag(image)
return image_repo_tags[image] return image_repo_tags[image]
@ -198,8 +199,9 @@ def get_image_latest_tag(image):
if res.ok: if res.ok:
break break
except requests.exceptions.Timeout: except requests.exceptions.Timeout:
LOG.warning("Failed to fetch url %s for %d/%d attempt(s)", LOG.warning(
url, attempt, max_attempts) "Failed to fetch url %s for %d/%d attempt(s)", url, attempt,
max_attempts)
time.sleep(5) time.sleep(5)
except requests.exceptions.TooManyRedirects: except requests.exceptions.TooManyRedirects:
logging.error("Failed to fetch url %s, TooManyRedirects", url) logging.error("Failed to fetch url %s, TooManyRedirects", url)
@ -208,18 +210,19 @@ def get_image_latest_tag(image):
logging.error("Failed to fetch url %s, error: %s", url, e) logging.error("Failed to fetch url %s, error: %s", url, e)
return 0 return 0
if attempt == max_attempts: if attempt == max_attempts:
logging.error("Failed to connect to quay.io for %d attempt(s)", logging.error(
attempt) "Failed to connect to quay.io for %d attempt(s)", attempt)
return 0 return 0
if res.status_code != 200: if res.status_code != 200:
logging.error("Image %s is not available on quay.io or " logging.error(
"requires authentication", image) "Image %s is not available on quay.io or "
"requires authentication", image)
return 0 return 0
try: try:
res = res.json() res = res.json()
except json.decoder.JSONDecodeError: # pylint: disable=no-member except json.decoder.JSONDecodeError: # pylint: disable=no-member
logging.error("Unable to parse response from quay.io (%s)", res.url) logging.error("Unable to parse response from quay.io (%s)", res.url)
return 0 return 0
@ -240,8 +243,9 @@ def get_image_latest_tag(image):
if tag_filter in tag["name"]: if tag_filter in tag["name"]:
return tag["name"] return tag["name"]
LOG.info("Skipping tag %s as not matching to the filter %s", LOG.info(
tag["name"], tag_filter) "Skipping tag %s as not matching to the filter %s",
tag["name"], tag_filter)
if not possible_tag: if not possible_tag:
possible_tag = tag["name"] possible_tag = tag["name"]
@ -273,8 +277,9 @@ def traverse(obj, dict_path=None):
"""Accepts Python dictionary with values.yaml contents, """Accepts Python dictionary with values.yaml contents,
updates it with latest git commit id's. updates it with latest git commit id's.
""" """
LOG.debug("traverse: dict_path: %s, object type: %s, object: %s", LOG.debug(
dict_path, type(obj), obj) "traverse: dict_path: %s, object type: %s, object: %s", dict_path,
type(obj), obj)
if dict_path is None: if dict_path is None:
dict_path = [] dict_path = []
@ -293,27 +298,29 @@ def traverse(obj, dict_path=None):
git_url = v["location"] git_url = v["location"]
if skip_list and k in skip_list: if skip_list and k in skip_list:
LOG.info("Ignoring chart %s, it is in a " LOG.info(
"skip list (%s)", k, git_url) "Ignoring chart %s, it is in a "
"skip list (%s)", k, git_url)
continue continue
new_git_commit_id = get_commit_id(git_url) new_git_commit_id = get_commit_id(git_url)
# Update git commit id in reference field of dictionary # Update git commit id in reference field of dictionary
if old_git_commit_id != new_git_commit_id: if old_git_commit_id != new_git_commit_id:
LOG.info("Updating git reference for " LOG.info(
"chart %s from %s to %s (%s)", "Updating git reference for "
k, old_git_commit_id, new_git_commit_id, "chart %s from %s to %s (%s)", k, old_git_commit_id,
git_url) new_git_commit_id, git_url)
v["reference"] = new_git_commit_id v["reference"] = new_git_commit_id
else: else:
LOG.info("Git reference %s for chart %s is already " LOG.info(
"up to date (%s)", "Git reference %s for chart %s is already "
old_git_commit_id, k, git_url) "up to date (%s)", old_git_commit_id, k, git_url)
else: else:
LOG.debug("value %s inside object is not a dictionary, " LOG.debug(
"or it does not contain key \"type\" with " "value %s inside object is not a dictionary, "
"value \"git\", skipping", v) "or it does not contain key \"type\" with "
"value \"git\", skipping", v)
# Traverse one level deeper # Traverse one level deeper
traverse(v, dict_path + [k]) traverse(v, dict_path + [k])
@ -336,8 +343,7 @@ def traverse(obj, dict_path=None):
if isinstance(v, str): if isinstance(v, str):
for image_repo in image_repo_git_url: for image_repo in image_repo_git_url:
if image_repo in v: if image_repo in v:
LOG.debug("image_repo %s is in %s string", LOG.debug("image_repo %s is in %s string", image_repo, v)
image_repo, v)
# hash_v: {"&whatever repo_url", "git commit id tag"} # hash_v: {"&whatever repo_url", "git commit id tag"}
# Note: "image" below could contain not just image, # Note: "image" below could contain not just image,
@ -346,8 +352,9 @@ def traverse(obj, dict_path=None):
image, old_image_tag = hash_v image, old_image_tag = hash_v
if skip_list and image in skip_list: if skip_list and image in skip_list:
LOG.info("Ignoring image %s, it is in a " LOG.info(
"skip list", image) "Ignoring image %s, it is in a "
"skip list", image)
continue continue
new_image_tag = get_image_tag(image) new_image_tag = get_image_tag(image)
@ -357,19 +364,23 @@ def traverse(obj, dict_path=None):
# Update git commit id in tag of container image # Update git commit id in tag of container image
if old_image_tag != new_image_tag: if old_image_tag != new_image_tag:
LOG.info("Updating git commit id in " LOG.info(
"tag of container image %s from %s to %s", "Updating git commit id in "
image, old_image_tag, new_image_tag) "tag of container image %s from %s to %s", image,
set_by_path(versions_data_dict, dict_path, old_image_tag, new_image_tag)
image + ":" + new_image_tag) set_by_path(
versions_data_dict, dict_path,
image + ":" + new_image_tag)
else: else:
LOG.info("Git tag %s for container " LOG.info(
"image %s is already up to date", "Git tag %s for container "
old_image_tag, image) "image %s is already up to date", old_image_tag,
image)
else: else:
LOG.debug("image_repo %s is not in %s string, " LOG.debug(
"skipping", image_repo, v) "image_repo %s is not in %s string, "
"skipping", image_repo, v)
else: else:
LOG.debug("value %s is not string, skipping", v) LOG.debug("value %s is not string, skipping", v)
@ -387,8 +398,8 @@ def print_versions_table():
table_format = "{:48s} {:60s} {:54s} {:41s}\n" table_format = "{:48s} {:60s} {:54s} {:41s}\n"
table_content = "\n" table_content = "\n"
table_content += table_format.format("Image repo","Git repo", table_content += table_format.format(
"Image repo tag","Git repo Commit ID") "Image repo", "Git repo", "Image repo tag", "Git repo Commit ID")
# Copy dicts for later modification # Copy dicts for later modification
image_repo_tags_copy = copy.deepcopy(image_repo_tags) image_repo_tags_copy = copy.deepcopy(image_repo_tags)
@ -408,10 +419,9 @@ def print_versions_table():
if not git_repo in git_url_commit_ids_copy: if not git_repo in git_url_commit_ids_copy:
git_url_commit_ids_copy[git_repo] = lsremote(git_repo, "HEAD") git_url_commit_ids_copy[git_repo] = lsremote(git_repo, "HEAD")
table_content += table_format.format(image_repo, table_content += table_format.format(
git_repo, image_repo, git_repo, image_repo_tags_copy[image_repo],
image_repo_tags_copy[image_repo], git_url_commit_ids_copy[git_repo])
git_url_commit_ids_copy[git_repo])
LOG.info("") LOG.info("")
for line in table_content.splitlines(): for line in table_content.splitlines():
@ -447,12 +457,14 @@ def print_missing_references():
for ref in missing_references: for ref in missing_references:
LOG.warning(missing_references[ref]) LOG.warning(missing_references[ref])
LOG.warning("") LOG.warning("")
LOG.warning("Refs which are not in git_url_commit_ids mean that " LOG.warning(
"we have not been updating chart references (or " "Refs which are not in git_url_commit_ids mean that "
"there are no charts referred in versions.yaml)") "we have not been updating chart references (or "
LOG.warning("Refs which are not in image_repo_tags mean that we " "there are no charts referred in versions.yaml)")
"have not been updating image tags (or there are no " LOG.warning(
"images referred in versions.yaml)") "Refs which are not in image_repo_tags mean that we "
"have not been updating image tags (or there are no "
"images referred in versions.yaml)")
LOG.warning("") LOG.warning("")
@ -483,7 +495,8 @@ def print_outdated_images():
# This is where we check if there is tag matching commit_id exists, # This is where we check if there is tag matching commit_id exists,
# and if not, then we append that image_repo to the list of # and if not, then we append that image_repo to the list of
# possibly outdated images # possibly outdated images
if git_url_commit_ids_copy[git_repo] not in image_repo_tags_copy[image_repo]: if git_url_commit_ids_copy[git_repo] not in image_repo_tags_copy[
image_repo]:
possibly_outdated_images.append(image_repo) possibly_outdated_images.append(image_repo)
if possibly_outdated_images: if possibly_outdated_images:
@ -498,20 +511,26 @@ if __name__ == "__main__":
"""Main program """Main program
""" """
parser.add_argument("--in-file", default="versions.yaml", parser.add_argument(
help="/path/to/versions.yaml input file; " "--in-file",
"default - \"./versions.yaml\"") default="versions.yaml",
parser.add_argument("--out-file", default="versions.yaml", help="/path/to/versions.yaml input file; "
help="name of output file; default - " "default - \"./versions.yaml\"")
"\"versions.yaml\" (overwrite existing)") parser.add_argument(
parser.add_argument("--skip", "--out-file",
help="comma-delimited list of images and charts " default="versions.yaml",
"to skip during the update; e.g. \"ceph\" " help="name of output file; default - "
"will skip all charts and images which have " "\"versions.yaml\" (overwrite existing)")
"\"ceph\" in the name") parser.add_argument(
parser.add_argument('--tag-filter', "--skip",
help="e.g. \"ubuntu\"; update would use image ref. " help="comma-delimited list of images and charts "
"tags on quay.io matching the filter") "to skip during the update; e.g. \"ceph\" "
"will skip all charts and images which have "
"\"ceph\" in the name")
parser.add_argument(
'--tag-filter',
help="e.g. \"ubuntu\"; update would use image ref. "
"tags on quay.io matching the filter")
args = parser.parse_args() args = parser.parse_args()
in_file = args.in_file in_file = args.in_file
@ -526,15 +545,16 @@ if __name__ == "__main__":
LOG.info("Tag filter: %s", tag_filter) LOG.info("Tag filter: %s", tag_filter)
if os.path.basename(out_file) != out_file: if os.path.basename(out_file) != out_file:
logging.error("Name of the output file must not contain path, " logging.error(
"but only the file name.") "Name of the output file must not contain path, "
"but only the file name.")
print("\n") print("\n")
parser.print_help() parser.print_help()
sys.exit(1) sys.exit(1)
if os.path.isfile(in_file): if os.path.isfile(in_file):
out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), out_file = os.path.join(
out_file) os.path.dirname(os.path.abspath(in_file)), out_file)
with open(in_file, "r") as f: with open(in_file, "r") as f:
f_old = f.read() f_old = f.read()
versions_data_dict = yaml.safe_load(f_old) versions_data_dict = yaml.safe_load(f_old)
@ -554,8 +574,11 @@ if __name__ == "__main__":
with open(out_file, "w") as f: with open(out_file, "w") as f:
if os.path.samefile(in_file, out_file): if os.path.samefile(in_file, out_file):
LOG.info("Overwriting %s", in_file) LOG.info("Overwriting %s", in_file)
f.write(yaml.safe_dump(versions_data_dict, f.write(
default_flow_style=False, yaml.safe_dump(
explicit_end=True, explicit_start=True, versions_data_dict,
width=4096)) default_flow_style=False,
explicit_end=True,
explicit_start=True,
width=4096))
LOG.info("New versions.yaml created as %s", out_file) LOG.info("New versions.yaml created as %s", out_file)

35
tox.ini
View File

@ -1,6 +1,8 @@
[tox] [tox]
# Allows docs to be built without setup.py having to exist. Requires that # Allows docs to be built without setup.py having to exist. Requires that
# usedevelop be False as well (which it is by default). # usedevelop be False as well (which it is by default).
envlist = pep8
minversion = 2.3.1
skipsdist = True skipsdist = True
[testenv] [testenv]
@ -15,7 +17,38 @@ commands = {posargs}
[testenv:docs] [testenv:docs]
basepython = python3 basepython = python3
whitelist_externals = rm whitelist_externals = rm
deps = -r{toxinidir}/doc/requirements.txt deps =
-r{toxinidir}/doc/requirements.txt
commands = commands =
rm -rf doc/build rm -rf doc/build
sphinx-build -W -b html doc/source doc/build/html sphinx-build -W -b html doc/source doc/build/html
[testenv:fmt]
basepython = python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
yapf -ir {toxinidir}/tools
[testenv:pep8]
basepython = python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
bandit -r {toxinidir}/tools -n 5
flake8 {toxinidir}/tools
yapf -dr {toxinidir}/tools
[flake8]
filename = *.py
show-source = true
# [H106] Don't put vim configuration in source files.
# [H201] No 'except:' at least use 'except Exception:'
# [H904] Delay string interpolations at logging calls.
enable-extensions = H106,H201,H904
# [W503] line break before binary operator
ignore = W503
exclude=.venv,.git,.tox,build,dist,*lib/python*,*egg,tools,*.ini,*.po,*.pot
max-complexity = 24
application-import-names = treasuremap
import-order-style = pep8