Update App Catalog dead link checker

This commit changes the periodic script that checks asset URLs to
verify they return a 200, and notes the ones that fail. This update
causes the script to output a new file (assets_dead.yaml) rather than
modifying the original assets.yaml file.

Change-Id: If5bb42414555a8f94f5472acd7d10f56c084f4c5

jenkins/scripts/check_app_catalog_yaml.py

@@ -1,6 +1,4 @@
-#! /usr/bin/env python
+#!/usr/bin/env python
-#
-# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,118 +18,21 @@ import requestsexceptions
 import yaml
 
 
-from collections import OrderedDict
-
-
-def project_representer(dumper, data):
-    return dumper.represent_mapping('tag:yaml.org,2002:map',
-                                    data.items())
-
-
-def construct_yaml_map(self, node):
-    data = OrderedDict()
-    yield data
-    value = self.construct_mapping(node)
-
-    if isinstance(node, yaml.MappingNode):
-        self.flatten_mapping(node)
-    else:
-        raise yaml.constructor.ConstructorError(
-            None, None,
-            'expected a mapping node, but found %s' % node.id,
-            node.start_mark)
-
-    mapping = OrderedDict()
-    for key_node, value_node in node.value:
-        key = self.construct_object(key_node, deep=False)
-        try:
-            hash(key)
-        except TypeError as exc:
-            raise yaml.constructor.ConstructorError(
-                'while constructing a mapping', node.start_mark,
-                'found unacceptable key (%s)' % exc, key_node.start_mark)
-        value = self.construct_object(value_node, deep=False)
-        mapping[key] = value
-    data.update(mapping)
-
-
-class IndentedEmitter(yaml.emitter.Emitter):
-    def expect_block_sequence(self):
-        self.increase_indent(flow=False, indentless=False)
-        self.state = self.expect_first_block_sequence_item
-
-
-class IndentedDumper(IndentedEmitter, yaml.serializer.Serializer,
-                     yaml.representer.Representer, yaml.resolver.Resolver):
-    def __init__(self, stream,
-                 default_style=None, default_flow_style=None,
-                 canonical=None, indent=None, width=None,
-                 allow_unicode=None, line_break=None,
-                 encoding=None, explicit_start=None, explicit_end=None,
-                 version=None, tags=None):
-        IndentedEmitter.__init__(
-            self, stream, canonical=canonical,
-            indent=indent, width=width,
-            allow_unicode=allow_unicode,
-            line_break=line_break)
-        yaml.serializer.Serializer.__init__(
-            self, encoding=encoding,
-            explicit_start=explicit_start,
-            explicit_end=explicit_end,
-            version=version, tags=tags)
-        yaml.representer.Representer.__init__(
-            self, default_style=default_style,
-            default_flow_style=default_flow_style)
-        yaml.resolver.Resolver.__init__(self)
-
-
-def get_hashes(hash_url):
-    hashes = {}
-    r = requests.get(hash_url, allow_redirects=True)
-    if r.status_code == 200:
-        for line in r.iter_lines():
-            try:
-                hash, file = line.split("  ")
-            except ValueError:
-                continue
-            hashes[file] = hash
-    return hashes
-
-
 def main():
     requestsexceptions.squelch_warnings()
-    yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-                         construct_yaml_map)
-
-    yaml.add_representer(OrderedDict, project_representer,
-                         Dumper=IndentedDumper)
-
     data = yaml.load(open('openstack_catalog/web/static/assets.yaml'))
 
-    assets = []
+    assets = {}
     for a in data['assets']:
         url = a.get('attributes', {}).get('url')
-        if not a.get('active', True) or not url:
+        if url:
-            assets.append(a)
+            r = requests.head(url, allow_redirects=True)
-            continue
+            if r.status_code != 200:
+                assets[a['name']] = {'active': False}
 
-        r = requests.head(url, allow_redirects=True)
+    with open('openstack_catalog/web/static/assets_dead.yaml', 'w') as out:
-        if r.status_code != 200:
+        out.write(yaml.safe_dump({"assets": assets}))
-            a['active'] = False
-        else:
-            hash_url = a.get('hash_url')
-            if hash_url:
-                hashes = get_hashes(hash_url)
-                filename = url.split("/")[-1]
-                a['attributes']['hash'] = hashes.get(filename, 'unknown')
 
-        assets.append(a)
-
-    output = {'assets': assets}
-    with open('openstack_catalog/web/static/assets.yaml', 'w') as out:
-        out.write(yaml.dump(output, default_flow_style=False,
-                            Dumper=IndentedDumper, width=80,
-                            indent=2))
 
 if __name__ == '__main__':
     main()