Browse Source

Standardize Treasuremap code with YAPF

From recently merged document updates in [0] there is a desire to
standardize the Airship project python codebase.  This is the effort
to do so for the Treasuremap project.

[0] https://review.opendev.org/#/c/671291/

Change-Id: Icd45f1b99a90e6c934a84fdd91f2f7f8af5a8ddb
tags/v1.3
HUGHES, ALEXANDER (ah8742) 2 months ago
parent
commit
8d0b847a03
6 changed files with 171 additions and 95 deletions
  1. 2
    0
      .gitignore
  2. 10
    0
      .style.yapf
  3. 5
    0
      Makefile
  4. 3
    0
      test-requirements.txt
  5. 117
    94
      tools/updater.py
  6. 34
    1
      tox.ini

+ 2
- 0
.gitignore View File

@@ -0,0 +1,2 @@
1
+# Unit test / coverage reports
2
+.tox/

+ 10
- 0
.style.yapf View File

@@ -0,0 +1,10 @@
1
+[style]
2
+based_on_style = pep8
3
+spaces_before_comment = 2
4
+column_limit = 79
5
+blank_line_before_nested_class_or_def = false
6
+blank_line_before_module_docstring = true
7
+split_before_logical_operator = true
8
+split_before_first_argument = true
9
+allow_split_before_dict_value = false
10
+split_before_arithmetic_operator = true

+ 5
- 0
Makefile View File

@@ -25,3 +25,8 @@ docs: clean build_docs
25 25
 .PHONY: build_docs
26 26
 build_docs:
27 27
 	tox -e docs
28
+
29
+# Perform auto formatting
30
+.PHONY: format
31
+format:
32
+	tox -e fmt

+ 3
- 0
test-requirements.txt View File

@@ -0,0 +1,3 @@
1
+yapf==0.27.0
2
+flake8-import-order==0.18.1
3
+bandit==1.6.0

+ 117
- 94
tools/updater.py View File

@@ -42,15 +42,17 @@ try:
42 42
     import git
43 43
     import yaml
44 44
 except ImportError as e:
45
-    sys.exit("Failed to import git/yaml libraries needed to run "
46
-             "this tool %s" % str(e))
47
-
48
-descr_text = ("Being run in directory with versions.yaml, will create "
49
-              "versions.new.yaml, with updated git commit id's to the "
50
-              "latest HEAD in references of all charts. In addition to "
51
-              "that, the tool updates references to the container images "
52
-              "with the tag, equal to the latest image which exists on "
53
-              "quay.io repository and is available for download.")
45
+    sys.exit(
46
+        "Failed to import git/yaml libraries needed to run "
47
+        "this tool %s" % str(e))
48
+
49
+descr_text = (
50
+    "Being run in directory with versions.yaml, will create "
51
+    "versions.new.yaml, with updated git commit id's to the "
52
+    "latest HEAD in references of all charts. In addition to "
53
+    "that, the tool updates references to the container images "
54
+    "with the tag, equal to the latest image which exists on "
55
+    "quay.io repository and is available for download.")
54 56
 parser = argparse.ArgumentParser(description=descr_text)
55 57
 
56 58
 # Dictionary containing container image repository url to git url mapping
@@ -73,10 +75,8 @@ image_repo_git_url = {
73 75
     "quay.io/airshipit/drydock": "https://opendev.org/airship/drydock",
74 76
     # maas-{rack,region}-controller images are built
75 77
     # from airship-maas repository:
76
-    "quay.io/airshipit/maas-rack-controller":
77
-        "https://opendev.org/airship/maas",
78
-    "quay.io/airshipit/maas-region-controller":
79
-        "https://opendev.org/airship/maas",
78
+    "quay.io/airshipit/maas-rack-controller": "https://opendev.org/airship/maas",
79
+    "quay.io/airshipit/maas-region-controller": "https://opendev.org/airship/maas",
80 80
     "quay.io/airshipit/pegleg": "https://opendev.org/airship/pegleg",
81 81
     "quay.io/airshipit/promenade": "https://opendev.org/airship/promenade",
82 82
     "quay.io/airshipit/shipyard": "https://opendev.org/airship/shipyard",
@@ -147,9 +147,9 @@ def get_commit_id(url):
147 147
     # fetch latest commit ID and add new dictionary entry
148 148
     LOG.debug("git_url_commit_ids: %s", git_url_commit_ids)
149 149
     if url not in git_url_commit_ids:
150
-        LOG.debug("git url: %s " +
151
-                      "is not in git_url_commit_ids dict; "
152
-                      "adding it with HEAD commit id", url)
150
+        LOG.debug(
151
+            "git url: %s " + "is not in git_url_commit_ids dict; "
152
+            "adding it with HEAD commit id", url)
153 153
         git_url_commit_ids[url] = lsremote(url, "HEAD")
154 154
 
155 155
     return git_url_commit_ids[url]
@@ -160,18 +160,19 @@ def get_image_tag(image):
160 160
     returns 0 (image not hosted on quay.io), True, or False
161 161
     """
162 162
     if not image.startswith("quay.io/"):
163
-        LOG.info("Unable to verify if image %s "
164
-                     "is in containers repository: only quay.io is "
165
-                     "supported at the moment", image)
163
+        LOG.info(
164
+            "Unable to verify if image %s "
165
+            "is in containers repository: only quay.io is "
166
+            "supported at the moment", image)
166 167
         return 0
167 168
 
168 169
     # If we don't have this image in our images's dictionary,
169 170
     # fetch latest tag and add new dictionary entry
170 171
     LOG.debug("image_repo_tags: %s", image_repo_tags)
171 172
     if image not in image_repo_tags:
172
-        LOG.debug("image: %s " +
173
-                      "is not in image_repo_tags dict; "
174
-                      "adding it with latest tag", image)
173
+        LOG.debug(
174
+            "image: %s " + "is not in image_repo_tags dict; "
175
+            "adding it with latest tag", image)
175 176
         image_repo_tags[image] = get_image_latest_tag(image)
176 177
 
177 178
     return image_repo_tags[image]
@@ -198,8 +199,9 @@ def get_image_latest_tag(image):
198 199
             if res.ok:
199 200
                 break
200 201
         except requests.exceptions.Timeout:
201
-            LOG.warning("Failed to fetch url %s for %d/%d attempt(s)",
202
-                             url, attempt, max_attempts)
202
+            LOG.warning(
203
+                "Failed to fetch url %s for %d/%d attempt(s)", url, attempt,
204
+                max_attempts)
203 205
             time.sleep(5)
204 206
         except requests.exceptions.TooManyRedirects:
205 207
             logging.error("Failed to fetch url %s, TooManyRedirects", url)
@@ -208,18 +210,19 @@ def get_image_latest_tag(image):
208 210
             logging.error("Failed to fetch url %s, error: %s", url, e)
209 211
             return 0
210 212
     if attempt == max_attempts:
211
-        logging.error("Failed to connect to quay.io for %d attempt(s)",
212
-                       attempt)
213
+        logging.error(
214
+            "Failed to connect to quay.io for %d attempt(s)", attempt)
213 215
         return 0
214 216
 
215 217
     if res.status_code != 200:
216
-        logging.error("Image %s is not available on quay.io or "
217
-                      "requires authentication", image)
218
+        logging.error(
219
+            "Image %s is not available on quay.io or "
220
+            "requires authentication", image)
218 221
         return 0
219 222
 
220 223
     try:
221 224
         res = res.json()
222
-    except json.decoder.JSONDecodeError: # pylint: disable=no-member
225
+    except json.decoder.JSONDecodeError:  # pylint: disable=no-member
223 226
         logging.error("Unable to parse response from quay.io (%s)", res.url)
224 227
         return 0
225 228
 
@@ -240,8 +243,9 @@ def get_image_latest_tag(image):
240 243
             if tag_filter in tag["name"]:
241 244
                 return tag["name"]
242 245
 
243
-            LOG.info("Skipping tag %s as not matching to the filter %s",
244
-                        tag["name"], tag_filter)
246
+            LOG.info(
247
+                "Skipping tag %s as not matching to the filter %s",
248
+                tag["name"], tag_filter)
245 249
             if not possible_tag:
246 250
                 possible_tag = tag["name"]
247 251
 
@@ -273,8 +277,9 @@ def traverse(obj, dict_path=None):
273 277
     """Accepts Python dictionary with values.yaml contents,
274 278
     updates it with latest git commit id's.
275 279
     """
276
-    LOG.debug("traverse: dict_path: %s, object type: %s, object: %s",
277
-                  dict_path, type(obj), obj)
280
+    LOG.debug(
281
+        "traverse: dict_path: %s, object type: %s, object: %s", dict_path,
282
+        type(obj), obj)
278 283
 
279 284
     if dict_path is None:
280 285
         dict_path = []
@@ -293,27 +298,29 @@ def traverse(obj, dict_path=None):
293 298
                 git_url = v["location"]
294 299
 
295 300
                 if skip_list and k in skip_list:
296
-                    LOG.info("Ignoring chart %s, it is in a "
297
-                                 "skip list (%s)", k, git_url)
301
+                    LOG.info(
302
+                        "Ignoring chart %s, it is in a "
303
+                        "skip list (%s)", k, git_url)
298 304
                     continue
299 305
 
300 306
                 new_git_commit_id = get_commit_id(git_url)
301 307
 
302 308
                 # Update git commit id in reference field of dictionary
303 309
                 if old_git_commit_id != new_git_commit_id:
304
-                    LOG.info("Updating git reference for "
305
-                                 "chart %s from %s to %s (%s)",
306
-                                  k, old_git_commit_id, new_git_commit_id,
307
-                                  git_url)
310
+                    LOG.info(
311
+                        "Updating git reference for "
312
+                        "chart %s from %s to %s (%s)", k, old_git_commit_id,
313
+                        new_git_commit_id, git_url)
308 314
                     v["reference"] = new_git_commit_id
309 315
                 else:
310
-                    LOG.info("Git reference %s for chart %s is already "
311
-                                 "up to date (%s)",
312
-                                  old_git_commit_id, k, git_url)
316
+                    LOG.info(
317
+                        "Git reference %s for chart %s is already "
318
+                        "up to date (%s)", old_git_commit_id, k, git_url)
313 319
             else:
314
-                LOG.debug("value %s inside object is not a dictionary, "
315
-                              "or it does not contain key \"type\" with "
316
-                              "value \"git\", skipping", v)
320
+                LOG.debug(
321
+                    "value %s inside object is not a dictionary, "
322
+                    "or it does not contain key \"type\" with "
323
+                    "value \"git\", skipping", v)
317 324
 
318 325
             # Traverse one level deeper
319 326
             traverse(v, dict_path + [k])
@@ -336,8 +343,7 @@ def traverse(obj, dict_path=None):
336 343
         if isinstance(v, str):
337 344
             for image_repo in image_repo_git_url:
338 345
                 if image_repo in v:
339
-                    LOG.debug("image_repo %s is in %s string",
340
-                                   image_repo, v)
346
+                    LOG.debug("image_repo %s is in %s string", image_repo, v)
341 347
 
342 348
                     # hash_v: {"&whatever repo_url", "git commit id tag"}
343 349
                     # Note: "image" below could contain not just image,
@@ -346,8 +352,9 @@ def traverse(obj, dict_path=None):
346 352
                     image, old_image_tag = hash_v
347 353
 
348 354
                     if skip_list and image in skip_list:
349
-                        LOG.info("Ignoring image %s, it is in a "
350
-                                    "skip list", image)
355
+                        LOG.info(
356
+                            "Ignoring image %s, it is in a "
357
+                            "skip list", image)
351 358
                         continue
352 359
 
353 360
                     new_image_tag = get_image_tag(image)
@@ -357,19 +364,23 @@ def traverse(obj, dict_path=None):
357 364
 
358 365
                     # Update git commit id in tag of container image
359 366
                     if old_image_tag != new_image_tag:
360
-                        LOG.info("Updating git commit id in "
361
-                                     "tag of container image %s from %s to %s",
362
-                                     image, old_image_tag, new_image_tag)
363
-                        set_by_path(versions_data_dict, dict_path,
364
-                                    image + ":" + new_image_tag)
367
+                        LOG.info(
368
+                            "Updating git commit id in "
369
+                            "tag of container image %s from %s to %s", image,
370
+                            old_image_tag, new_image_tag)
371
+                        set_by_path(
372
+                            versions_data_dict, dict_path,
373
+                            image + ":" + new_image_tag)
365 374
 
366 375
                     else:
367
-                        LOG.info("Git tag %s for container "
368
-                                     "image %s is already up to date",
369
-                                     old_image_tag, image)
376
+                        LOG.info(
377
+                            "Git tag %s for container "
378
+                            "image %s is already up to date", old_image_tag,
379
+                            image)
370 380
                 else:
371
-                    LOG.debug("image_repo %s is not in %s string, "
372
-                                  "skipping", image_repo, v)
381
+                    LOG.debug(
382
+                        "image_repo %s is not in %s string, "
383
+                        "skipping", image_repo, v)
373 384
         else:
374 385
             LOG.debug("value %s is not string, skipping", v)
375 386
 
@@ -387,8 +398,8 @@ def print_versions_table():
387 398
 
388 399
     table_format = "{:48s} {:60s} {:54s} {:41s}\n"
389 400
     table_content = "\n"
390
-    table_content += table_format.format("Image repo","Git repo",
391
-                                         "Image repo tag","Git repo Commit ID")
401
+    table_content += table_format.format(
402
+        "Image repo", "Git repo", "Image repo tag", "Git repo Commit ID")
392 403
 
393 404
     # Copy dicts for later modification
394 405
     image_repo_tags_copy = copy.deepcopy(image_repo_tags)
@@ -408,10 +419,9 @@ def print_versions_table():
408 419
         if not git_repo in git_url_commit_ids_copy:
409 420
             git_url_commit_ids_copy[git_repo] = lsremote(git_repo, "HEAD")
410 421
 
411
-        table_content += table_format.format(image_repo,
412
-                                             git_repo,
413
-                                             image_repo_tags_copy[image_repo],
414
-                                             git_url_commit_ids_copy[git_repo])
422
+        table_content += table_format.format(
423
+            image_repo, git_repo, image_repo_tags_copy[image_repo],
424
+            git_url_commit_ids_copy[git_repo])
415 425
 
416 426
     LOG.info("")
417 427
     for line in table_content.splitlines():
@@ -447,12 +457,14 @@ def print_missing_references():
447 457
         for ref in missing_references:
448 458
             LOG.warning(missing_references[ref])
449 459
         LOG.warning("")
450
-        LOG.warning("Refs which are not in git_url_commit_ids mean that "
451
-                        "we have not been updating chart references (or "
452
-                        "there are no charts referred in versions.yaml)")
453
-        LOG.warning("Refs which are not in image_repo_tags mean that we "
454
-                        "have not been updating image tags (or there are no "
455
-                        "images referred in versions.yaml)")
460
+        LOG.warning(
461
+            "Refs which are not in git_url_commit_ids mean that "
462
+            "we have not been updating chart references (or "
463
+            "there are no charts referred in versions.yaml)")
464
+        LOG.warning(
465
+            "Refs which are not in image_repo_tags mean that we "
466
+            "have not been updating image tags (or there are no "
467
+            "images referred in versions.yaml)")
456 468
         LOG.warning("")
457 469
 
458 470
 
@@ -483,7 +495,8 @@ def print_outdated_images():
483 495
         # This is where we check if there is tag matching commit_id exists,
484 496
         # and if not, then we append that image_repo to the list of
485 497
         # possibly outdated images
486
-        if git_url_commit_ids_copy[git_repo] not in image_repo_tags_copy[image_repo]:
498
+        if git_url_commit_ids_copy[git_repo] not in image_repo_tags_copy[
499
+                image_repo]:
487 500
             possibly_outdated_images.append(image_repo)
488 501
 
489 502
     if possibly_outdated_images:
@@ -498,20 +511,26 @@ if __name__ == "__main__":
498 511
     """Main program
499 512
     """
500 513
 
501
-    parser.add_argument("--in-file", default="versions.yaml",
502
-                        help="/path/to/versions.yaml input file; "
503
-                             "default - \"./versions.yaml\"")
504
-    parser.add_argument("--out-file", default="versions.yaml",
505
-                        help="name of output file; default - "
506
-                             "\"versions.yaml\" (overwrite existing)")
507
-    parser.add_argument("--skip",
508
-                        help="comma-delimited list of images and charts "
509
-                             "to skip during the update; e.g. \"ceph\" "
510
-                             "will skip all charts and images which have "
511
-                             "\"ceph\" in the name")
512
-    parser.add_argument('--tag-filter',
513
-                        help="e.g. \"ubuntu\"; update would use image ref. "
514
-                             "tags on quay.io matching the filter")
514
+    parser.add_argument(
515
+        "--in-file",
516
+        default="versions.yaml",
517
+        help="/path/to/versions.yaml input file; "
518
+        "default - \"./versions.yaml\"")
519
+    parser.add_argument(
520
+        "--out-file",
521
+        default="versions.yaml",
522
+        help="name of output file; default - "
523
+        "\"versions.yaml\" (overwrite existing)")
524
+    parser.add_argument(
525
+        "--skip",
526
+        help="comma-delimited list of images and charts "
527
+        "to skip during the update; e.g. \"ceph\" "
528
+        "will skip all charts and images which have "
529
+        "\"ceph\" in the name")
530
+    parser.add_argument(
531
+        '--tag-filter',
532
+        help="e.g. \"ubuntu\"; update would use image ref. "
533
+        "tags on quay.io matching the filter")
515 534
 
516 535
     args = parser.parse_args()
517 536
     in_file = args.in_file
@@ -526,15 +545,16 @@ if __name__ == "__main__":
526 545
     LOG.info("Tag filter: %s", tag_filter)
527 546
 
528 547
     if os.path.basename(out_file) != out_file:
529
-        logging.error("Name of the output file must not contain path, "
530
-                      "but only the file name.")
548
+        logging.error(
549
+            "Name of the output file must not contain path, "
550
+            "but only the file name.")
531 551
         print("\n")
532 552
         parser.print_help()
533 553
         sys.exit(1)
534 554
 
535 555
     if os.path.isfile(in_file):
536
-        out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)),
537
-                                out_file)
556
+        out_file = os.path.join(
557
+            os.path.dirname(os.path.abspath(in_file)), out_file)
538 558
         with open(in_file, "r") as f:
539 559
             f_old = f.read()
540 560
             versions_data_dict = yaml.safe_load(f_old)
@@ -554,8 +574,11 @@ if __name__ == "__main__":
554 574
     with open(out_file, "w") as f:
555 575
         if os.path.samefile(in_file, out_file):
556 576
             LOG.info("Overwriting %s", in_file)
557
-        f.write(yaml.safe_dump(versions_data_dict,
558
-                               default_flow_style=False,
559
-                               explicit_end=True, explicit_start=True,
560
-                               width=4096))
577
+        f.write(
578
+            yaml.safe_dump(
579
+                versions_data_dict,
580
+                default_flow_style=False,
581
+                explicit_end=True,
582
+                explicit_start=True,
583
+                width=4096))
561 584
         LOG.info("New versions.yaml created as %s", out_file)

+ 34
- 1
tox.ini View File

@@ -1,6 +1,8 @@
1 1
 [tox]
2 2
 # Allows docs to be built without setup.py having to exist. Requires that
3 3
 # usedevelop be False as well (which it is by default).
4
+envlist = pep8
5
+minversion = 2.3.1
4 6
 skipsdist = True
5 7
 
6 8
 [testenv]
@@ -15,7 +17,38 @@ commands = {posargs}
15 17
 [testenv:docs]
16 18
 basepython = python3
17 19
 whitelist_externals = rm
18
-deps = -r{toxinidir}/doc/requirements.txt
20
+deps =
21
+    -r{toxinidir}/doc/requirements.txt
19 22
 commands =
20 23
     rm -rf doc/build
21 24
     sphinx-build -W -b html doc/source doc/build/html
25
+
26
+[testenv:fmt]
27
+basepython = python3
28
+deps =
29
+    -r{toxinidir}/test-requirements.txt
30
+commands =
31
+    yapf -ir {toxinidir}/tools
32
+
33
+[testenv:pep8]
34
+basepython = python3
35
+deps =
36
+    -r{toxinidir}/test-requirements.txt
37
+commands =
38
+    bandit -r {toxinidir}/tools -n 5
39
+    flake8 {toxinidir}/tools
40
+    yapf -dr {toxinidir}/tools
41
+
42
+[flake8]
43
+filename = *.py
44
+show-source = true
45
+# [H106] Don't put vim configuration in source files.
46
+# [H201] No 'except:' at least use 'except Exception:'
47
+# [H904] Delay string interpolations at logging calls.
48
+enable-extensions = H106,H201,H904
49
+# [W503] line break before binary operator
50
+ignore = W503
51
+exclude=.venv,.git,.tox,build,dist,*lib/python*,*egg,tools,*.ini,*.po,*.pot
52
+max-complexity = 24
53
+application-import-names = treasuremap
54
+import-order-style = pep8

Loading…
Cancel
Save