Merge "Check for dead links in App Catalog"

jenkins/jobs/app-catalog.yaml

@@ -0,0 +1,12 @@
+- job:
+    name: propose-app-catalog-update
+    node: proposal
+
+    builders:
+      - link-logs
+      - branch-git-prep:
+          branch: master
+      - shell: /usr/local/jenkins/slave_scripts/propose_app_catalog_update.sh
+
+    publishers:
+      - console-log

jenkins/scripts/check_app_catalog_yaml.py

@@ -0,0 +1,114 @@
+#! /usr/bin/env python
+#
+# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import yaml
+import requests
+from collections import OrderedDict
+import requestsexceptions
+
+def project_representer(dumper, data):
+    return dumper.represent_mapping('tag:yaml.org,2002:map',
+                                    data.items())
+
+
+def construct_yaml_map(self, node):
+    data = OrderedDict()
+    yield data
+    value = self.construct_mapping(node)
+
+    if isinstance(node, yaml.MappingNode):
+        self.flatten_mapping(node)
+    else:
+        raise yaml.constructor.ConstructorError(
+            None, None,
+            'expected a mapping node, but found %s' % node.id,
+            node.start_mark)
+
+    mapping = OrderedDict()
+    for key_node, value_node in node.value:
+        key = self.construct_object(key_node, deep=False)
+        try:
+            hash(key)
+        except TypeError as exc:
+            raise yaml.constructor.ConstructorError(
+                'while constructing a mapping', node.start_mark,
+                'found unacceptable key (%s)' % exc, key_node.start_mark)
+        value = self.construct_object(value_node, deep=False)
+        mapping[key] = value
+    data.update(mapping)
+
+
+class IndentedEmitter(yaml.emitter.Emitter):
+    def expect_block_sequence(self):
+        self.increase_indent(flow=False, indentless=False)
+        self.state = self.expect_first_block_sequence_item
+
+
+class IndentedDumper(IndentedEmitter, yaml.serializer.Serializer,
+                     yaml.representer.Representer, yaml.resolver.Resolver):
+    def __init__(self, stream,
+                 default_style=None, default_flow_style=None,
+                 canonical=None, indent=None, width=None,
+                 allow_unicode=None, line_break=None,
+                 encoding=None, explicit_start=None, explicit_end=None,
+                 version=None, tags=None):
+        IndentedEmitter.__init__(
+            self, stream, canonical=canonical,
+            indent=indent, width=width,
+            allow_unicode=allow_unicode,
+            line_break=line_break)
+        yaml.serializer.Serializer.__init__(
+            self, encoding=encoding,
+            explicit_start=explicit_start,
+            explicit_end=explicit_end,
+            version=version, tags=tags)
+        yaml.representer.Representer.__init__(
+            self, default_style=default_style,
+            default_flow_style=default_flow_style)
+        yaml.resolver.Resolver.__init__(self)
+
+
+def main():
+    requestsexceptions.squelch_warnings()
+    yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+                         construct_yaml_map)
+
+    yaml.add_representer(OrderedDict, project_representer,
+                         Dumper=IndentedDumper)
+
+    data = yaml.load(open('openstack_catalog/web/static/assets.yaml'))
+
+    assets = []
+    for a in data['assets']:
+        if not a.get('attributes', {}).get('active', True):
+            assets.append(a)
+            continue
+        url = a.get('attributes', {}).get('url')
+        if url:
+            r = requests.head(url, allow_redirects=True)
+            if r.status_code != 200:
+                a['attributes']['active'] = False
+        assets.append(a)
+
+    output = {'assets': assets}
+    with open('openstack_catalog/web/static/assets.yaml', 'w') as out:
+        out.write(yaml.dump(output, default_flow_style=False,
+                            Dumper=IndentedDumper, width=80,
+                            indent=2))
+
+if __name__ == '__main__':
+    main()

jenkins/scripts/propose_app_catalog_update.sh

@@ -0,0 +1,66 @@
+#!/bin/bash -xe
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+PROJECT="openstack/app-catalog"
+BRANCH="proposals"
+INITIAL_COMMIT_MSG="Detect dead links"
+TOPIC="detect-dead-links"
+USERNAME="proposal-bot"
+SUCCESS=0
+
+setup_git
+
+change_id=""
+# See if there is an open change in the detect-dead-links topic
+# If so, get the change id for the existing change for use in the
+# commit msg.
+change_info=$(ssh -p 29418 $USERNAME@review.openstack.org gerrit query --current-patch-set status:open project:$PROJECT topic:$TOPIC owner:$USERNAME)
+previous=$(echo "$change_info" | grep "^  number:" | awk '{print $2}')
+if [ -n "$previous" ]; then
+    change_id=$(echo "$change_info" | grep "^change" | awk '{print $2}')
+    # read return a non zero value when it reaches EOF. Because we use a
+    # heredoc here it will always reach EOF and return a nonzero value.
+    # Disable -e temporarily to get around the read.
+    # The reason we use read is to allow for multiline variable content
+    # and variable interpolation. Simply double quoting a string across
+    # multiple lines removes the newlines.
+    set +e
+    read -d '' COMMIT_MSG <<EOF
+$INITIAL_COMMIT_MSG
+
+Change-Id: $change_id
+EOF
+    set -e
+else
+    COMMIT_MSG=$INITIAL_COMMIT_MSG
+fi
+
+git review -s
+python /usr/local/jenkins/slave_scripts/check_app_catalog_yaml.py
+
+if ! git diff --stat --exit-code HEAD ; then
+    # Commit and review
+    git_args="-a -F-"
+    git commit $git_args <<EOF
+$COMMIT_MSG
+EOF
+    # Do error checking manually to ignore one class of failure.
+    set +e
+    OUTPUT=$(git review -t $TOPIC)
+    RET=$?
+    [[ "$RET" -eq "0" || "$OUTPUT" =~ "no new changes" || "$OUTPUT" =~ "no changes made" ]]
+    SUCCESS=$?
+fi
+
+exit $SUCCESS

zuul/layout.yaml

@@ -3664,6 +3664,8 @@ projects:
       - gate-app-catalog-npm-run-lint
     gate:
       - gate-app-catalog-npm-run-lint
+    experimental:
+      - propose-app-catalog-update
 
   - name: openstack/app-catalog-common
     template: