Browse Source

Merge "Add cache for results of requests to quay.io in Updater tool"

Zuul 3 weeks ago
parent
commit
82b2e57147
1 changed files with 75 additions and 44 deletions
  1. 75
    44
      tools/updater.py

+ 75
- 44
tools/updater.py View File

@@ -44,7 +44,7 @@ except ImportError as e:
44 44
     sys.exit("Failed to import git/yaml libraries needed to run" +
45 45
              "this tool %s" % str(e))
46 46
 
47
-descr_text="Being run in directory with versions.yaml, will create \
47
+descr_text = "Being run in directory with versions.yaml, will create \
48 48
             versions.new.yaml, with updated git commit id's to the \
49 49
             latest HEAD in references of all charts. In addition to \
50 50
             that, the tool updates references to the container images \
@@ -78,7 +78,8 @@ image_repo_git_url = {
78 78
     # sstream-cache image is built from airship-maas repository
79 79
     'quay.io/airshipit/sstream-cache': 'https://git.openstack.org/openstack/airship-maas',
80 80
     'quay.io/attcomdev/nagios': 'https://github.com/att-comdev/nagios',
81
-    'quay.io/attcomdev/prometheus-openstack-exporter': 'https://github.com/att-comdev/prometheus-openstack-exporter'
81
+    'quay.io/attcomdev/prometheus-openstack-exporter':
82
+        'https://github.com/att-comdev/prometheus-openstack-exporter'
82 83
 }
83 84
 
84 85
 logging.basicConfig(level=logging.INFO)
@@ -129,7 +130,7 @@ def lsremote(url, remote_ref):
129 130
     """Accepts git url and remote reference, returns git commit id."""
130 131
     git_commit_id_remote_ref = {}
131 132
     g = git.cmd.Git()
132
-    logging.info('Fetching ' + url + ' ' + remote_ref + ' reference...')
133
+    logging.info("Fetching %s %s reference...", url, remote_ref)
133 134
     hash_ref_list = g.ls_remote(url, remote_ref).split('\t')
134 135
     git_commit_id_remote_ref[hash_ref_list[1]] = hash_ref_list[0]
135 136
     return git_commit_id_remote_ref[remote_ref]
@@ -140,11 +141,10 @@ def get_commit_id(url):
140 141
     # If we don't have this git url in our url's dictionary,
141 142
     # fetch latest commit ID and add new dictionary entry
142 143
     logging.debug('git_url_commit_ids: %s', git_url_commit_ids)
143
-    logging.debug('image_repo_status: %s', image_repo_status)
144 144
     if url not in git_url_commit_ids:
145
-        logging.debug('git url: ' + url +
146
-                      ' is not in git_url_commit_ids dict;' +
147
-                      ' adding it with HEAD commit id')
145
+        logging.debug("git url: %s" +
146
+                      " is not in git_url_commit_ids dict;" +
147
+                      " adding it with HEAD commit id", url)
148 148
         git_url_commit_ids[url] = lsremote(url, 'HEAD')
149 149
 
150 150
     return git_url_commit_ids[url]
@@ -155,41 +155,64 @@ def get_image_tag(image):
155 155
     returns 0 (image not hosted on quay.io), True, or False
156 156
     """
157 157
     if not image.startswith('quay.io/'):
158
-        logging.info('Unable to verify if image ' + image +
159
-                     ' is in containers repository: only quay.io is' +
160
-                     ' supported at the moment')
158
+        logging.info("Unable to verify if image %s" +
159
+                     " is in containers repository: only quay.io is" +
160
+                     " supported at the moment", image)
161 161
         return 0
162 162
 
163
-    logging.info('Getting latest tag for image %s' % image)
163
+    # If we don't have this image in our images's dictionary,
164
+    # fetch latest tag and add new dictionary entry
165
+    logging.debug('image_repo_status: %s', image_repo_status)
166
+    if image not in image_repo_status:
167
+        logging.debug("image: %s" +
168
+                      " is not in image_repo_status dict;" +
169
+                      " adding it with latest tag", image)
170
+        image_repo_status[image] = get_image_latest_tag(image)
171
+
172
+    return image_repo_status[image]
173
+
174
+
175
+def get_image_latest_tag(image):
176
+    """Get latest image tag from quay.io,
177
+    returns latest image tag string, or 0 if a problem occured.
178
+    """
164 179
 
165
-    retries = 0
166
-    max_retries = 5
180
+    attempt = 0
181
+    max_attempts = 5
167 182
 
168 183
     hash_image = image.split('/')
169
-    url = 'https://quay.io/api/v1/repository/' + \
170
-    hash_image[1] + '/' + hash_image[2] + '/tag'
184
+    url = 'https://quay.io/api/v1/repository/{}/{}/tag/'
185
+    url = url.format(hash_image[1], hash_image[2])
186
+    logging.info("Fetching latest tag for image %s (%s)...", image, url)
171 187
 
172
-    while retries < max_retries:
173
-        retries = retries + 1
188
+    while attempt < max_attempts:
189
+        attempt = attempt + 1
174 190
         try:
175
-            res = requests.get(url, timeout = 5)
191
+            res = requests.get(url, timeout=5)
176 192
             if res.ok:
177 193
                 break
178 194
         except requests.exceptions.Timeout:
179
-            logging.warning("Failed to fetch url %s" % res.url)
195
+            logging.warning("Failed to fetch url %s for %d attempt(s)", url, attempt)
180 196
             time.sleep(1)
181
-    if retries == max_retries:
182
-        logging.error("Failed to connect to quay.io")
197
+        except requests.exceptions.TooManyRedirects:
198
+            logging.error("Failed to fetch url %s, TooManyRedirects", url)
199
+            return 0
200
+        except requests.exceptions.RequestException as e:
201
+            logging.error("Failed to fetch url %s, error: %s", url, e)
202
+            return 0
203
+    if attempt == max_attempts:
204
+        logging.error("Failed to connect to quay.io for %d attempt(s)", attempt)
183 205
         return 0
184 206
 
185 207
     if res.status_code != 200:
186
-        logging.error('Image %s is not available on quay.io or ' +
187
-                      'requires authentication', image)
208
+        logging.error("Image %s is not available on quay.io or " +
209
+                      "requires authentication", image)
210
+        return 0
188 211
 
189 212
     try:
190 213
         res = res.json()
191 214
     except json.decoder.JSONDecodeError: # pylint: disable=no-member
192
-        logging.error('Unable to parse response from quay.io (%s)' % res.url)
215
+        logging.error("Unable to parse response from quay.io (%s)", res.url)
193 216
         return 0
194 217
 
195 218
     try:
@@ -198,10 +221,10 @@ def get_image_tag(image):
198 221
                 if tag['name'] != 'master' and tag['name'] != 'latest':
199 222
                     return tag['name']
200 223
     except KeyError:
201
-        logging.error('Unable to parse response from quay.io (%s)' % res.url)
224
+        logging.error("Unable to parse response from quay.io (%s)", res.url)
202 225
         return 0
203 226
 
204
-    logging.error("Image with end_ts in path %s not found" % image)
227
+    logging.error("Image with end_ts in path %s not found", image)
205 228
     return 0
206 229
 
207 230
 
@@ -248,17 +271,16 @@ def traverse(obj, dict_path=None):
248 271
 
249 272
                 # Update git commit id in reference field of dictionary
250 273
                 if old_git_commit_id != new_git_commit_id:
251
-                    logging.info('Updating git reference for chart %s from %s to ' +
252
-                                 '%s (%s)',
274
+                    logging.info("Updating git reference for chart %s from %s to %s (%s)",
253 275
                                  k, old_git_commit_id, new_git_commit_id,
254 276
                                  git_url)
255 277
                     v['reference'] = new_git_commit_id
256 278
                 else:
257
-                    logging.info('Git reference %s for chart %s is already up to date (%s) ',
279
+                    logging.info("Git reference %s for chart %s is already up to date (%s)",
258 280
                                  old_git_commit_id, k, git_url)
259 281
             else:
260
-                logging.debug('value %s inside object is not a dictionary, or it does not ' +
261
-                              'contain key \'type\' with value \'git\', skipping', v)
282
+                logging.debug("value %s inside object is not a dictionary, or it does not " +
283
+                              "contain key \'type\' with value \'git\', skipping", v)
262 284
 
263 285
             # Traverse one level deeper
264 286
             traverse(v, dict_path + [k])
@@ -294,19 +316,19 @@ def traverse(obj, dict_path=None):
294 316
 
295 317
                     new_image_tag = get_image_tag(image)
296 318
                     if new_image_tag == 0:
297
-                        logging.error("Failed to get image tag for %s" % image)
319
+                        logging.error("Failed to get image tag for %s", image)
298 320
                         sys.exit(1)
299 321
 
300 322
                     # Update git commit id in tag of container image
301 323
                     if old_image_tag != new_image_tag:
302
-                        logging.info('Updating git commit id in ' +
303
-                                     'tag of container image %s from %s to %s',
324
+                        logging.info("Updating git commit id in " +
325
+                                     "tag of container image %s from %s to %s",
304 326
                                      image, old_image_tag, new_image_tag)
305 327
                         set_by_path(versions_data_dict, dict_path, image + ':' + new_image_tag)
306 328
 
307 329
                     else:
308
-                        logging.info('Git tag %s for container ' +
309
-                                     'image %s is already up to date',
330
+                        logging.info("Git tag %s for container " +
331
+                                     "image %s is already up to date",
310 332
                                      old_image_tag, image)
311 333
                 else:
312 334
                     logging.debug('image_repo %s is not in %s string, skipping', image_repo, v)
@@ -315,33 +337,42 @@ def traverse(obj, dict_path=None):
315 337
 
316 338
 
317 339
 if __name__ == '__main__':
318
-    """Small Main program"""
340
+    """Small Main program
341
+    """
319 342
 
320 343
     parser.add_argument('--in-file', default='versions.yaml',
321
-    help='/path/to/versions.yaml input file; default - "./versions.yaml"')
344
+                        help='/path/to/versions.yaml input file; default - "./versions.yaml"')
322 345
 
323 346
     parser.add_argument('--out-file', default='versions.yaml',
324
-    help='name of output file; default - "versions.yaml" (overwrite existing)')
347
+                        help='name of output file; default - "versions.yaml" (overwrite existing)')
325 348
 
326 349
     parser.add_argument('--skip',
327
-    help='comma-delimited list of images and charts to skip during the update')
350
+                        help='comma-delimited list of images and charts to skip during the update')
328 351
 
329 352
     args = parser.parse_args()
330 353
     in_file = args.in_file
331 354
     out_file = args.out_file
332 355
     if args.skip:
333 356
         skip_list = tuple(args.skip.strip().split(","))
334
-        logging.info('Skip list: %s', skip_list)
357
+        logging.info("Skip list: %s", skip_list)
335 358
     else:
336 359
         skip_list = None
337 360
 
361
+    if os.path.basename(out_file) != out_file:
362
+        logging.error("Name of the output file must not contain path, " +
363
+                      "but only the file name.")
364
+        print("\n")
365
+        parser.print_help()
366
+        sys.exit(1)
367
+
338 368
     if os.path.isfile(in_file):
339 369
         out_file = os.path.join(os.path.dirname(os.path.abspath(in_file)), out_file)
340 370
         with open(in_file, 'r') as f:
341 371
             f_old = f.read()
342 372
             versions_data_dict = yaml.safe_load(f_old)
343 373
     else:
344
-        logging.error("Can\'t find versions.yaml file.\n")
374
+        logging.error("Can\'t find versions.yaml file.")
375
+        print("\n")
345 376
         parser.print_help()
346 377
         sys.exit(1)
347 378
 
@@ -350,9 +381,9 @@ if __name__ == '__main__':
350 381
 
351 382
     with open(out_file, 'w') as f:
352 383
         if os.path.samefile(in_file, out_file):
353
-            logging.info('Overwriting %s' % in_file)
384
+            logging.info("Overwriting %s", in_file)
354 385
         f.write(yaml.safe_dump(versions_data_dict,
355 386
                                default_flow_style=False,
356 387
                                explicit_end=True, explicit_start=True,
357 388
                                width=4096))
358
-        logging.info('New versions.yaml created as %s' % out_file)
389
+        logging.info("New versions.yaml created as %s", out_file)

Loading…
Cancel
Save