From a4e0afd9ac90e4cd7279552facac9a034a790f34 Mon Sep 17 00:00:00 2001 From: Christopher Aedo Date: Thu, 17 Dec 2015 16:24:53 -0800 Subject: [PATCH] Check for dead links in App Catalog This commit includes a job meant to be run periodically which checks all links to assets listed in the Community App Catalog assets.yaml file. For any dead links, an additional attribute is added to the YAML file, and a review is submitted with the updated YAML which will prevent the link from being displayed until it's updated with a subsequent review. Change-Id: I991155edca8ed92431f7a5deacf2d3938cf70ada --- jenkins/jobs/app-catalog.yaml | 12 ++ jenkins/scripts/check_app_catalog_yaml.py | 114 ++++++++++++++++++ jenkins/scripts/propose_app_catalog_update.sh | 66 ++++++++++ zuul/layout.yaml | 2 + 4 files changed, 194 insertions(+) create mode 100644 jenkins/jobs/app-catalog.yaml create mode 100755 jenkins/scripts/check_app_catalog_yaml.py create mode 100755 jenkins/scripts/propose_app_catalog_update.sh diff --git a/jenkins/jobs/app-catalog.yaml b/jenkins/jobs/app-catalog.yaml new file mode 100644 index 0000000000..9f54b668c9 --- /dev/null +++ b/jenkins/jobs/app-catalog.yaml @@ -0,0 +1,12 @@ +- job: + name: propose-app-catalog-update + node: proposal + + builders: + - link-logs + - branch-git-prep: + branch: master + - shell: /usr/local/jenkins/slave_scripts/propose_app_catalog_update.sh + + publishers: + - console-log diff --git a/jenkins/scripts/check_app_catalog_yaml.py b/jenkins/scripts/check_app_catalog_yaml.py new file mode 100755 index 0000000000..7dec25ef2e --- /dev/null +++ b/jenkins/scripts/check_app_catalog_yaml.py @@ -0,0 +1,114 @@ +#! /usr/bin/env python +# +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import requests +from collections import OrderedDict +import requestsexceptions + +def project_representer(dumper, data): + return dumper.represent_mapping('tag:yaml.org,2002:map', + data.items()) + + +def construct_yaml_map(self, node): + data = OrderedDict() + yield data + value = self.construct_mapping(node) + + if isinstance(node, yaml.MappingNode): + self.flatten_mapping(node) + else: + raise yaml.constructor.ConstructorError( + None, None, + 'expected a mapping node, but found %s' % node.id, + node.start_mark) + + mapping = OrderedDict() + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=False) + try: + hash(key) + except TypeError as exc: + raise yaml.constructor.ConstructorError( + 'while constructing a mapping', node.start_mark, + 'found unacceptable key (%s)' % exc, key_node.start_mark) + value = self.construct_object(value_node, deep=False) + mapping[key] = value + data.update(mapping) + + +class IndentedEmitter(yaml.emitter.Emitter): + def expect_block_sequence(self): + self.increase_indent(flow=False, indentless=False) + self.state = self.expect_first_block_sequence_item + + +class IndentedDumper(IndentedEmitter, yaml.serializer.Serializer, + yaml.representer.Representer, yaml.resolver.Resolver): + def __init__(self, stream, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + IndentedEmitter.__init__( + self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, + line_break=line_break) + yaml.serializer.Serializer.__init__( + self, encoding=encoding, + explicit_start=explicit_start, + explicit_end=explicit_end, + version=version, tags=tags) + yaml.representer.Representer.__init__( + self, default_style=default_style, + default_flow_style=default_flow_style) + yaml.resolver.Resolver.__init__(self) + + +def main(): + requestsexceptions.squelch_warnings() + yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_yaml_map) + + yaml.add_representer(OrderedDict, project_representer, + Dumper=IndentedDumper) + + data = yaml.load(open('openstack_catalog/web/static/assets.yaml')) + + assets = [] + for a in data['assets']: + if not a.get('attributes', {}).get('active', True): + assets.append(a) + continue + url = a.get('attributes', {}).get('url') + if url: + r = requests.head(url, allow_redirects=True) + if r.status_code != 200: + a['attributes']['active'] = False + assets.append(a) + + output = {'assets': assets} + with open('openstack_catalog/web/static/assets.yaml', 'w') as out: + out.write(yaml.dump(output, default_flow_style=False, + Dumper=IndentedDumper, width=80, + indent=2)) + +if __name__ == '__main__': + main() diff --git a/jenkins/scripts/propose_app_catalog_update.sh b/jenkins/scripts/propose_app_catalog_update.sh new file mode 100755 index 0000000000..f2454d5eed --- /dev/null +++ b/jenkins/scripts/propose_app_catalog_update.sh @@ -0,0 +1,66 @@ +#!/bin/bash -xe + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +PROJECT="openstack/app-catalog" +BRANCH="proposals" +INITIAL_COMMIT_MSG="Detect dead links" +TOPIC="detect-dead-links" +USERNAME="proposal-bot" +SUCCESS=0 + +setup_git + +change_id="" +# See if there is an open change in the detect-dead-links topic +# If so, get the change id for the existing change for use in the +# commit msg. +change_info=$(ssh -p 29418 $USERNAME@review.openstack.org gerrit query --current-patch-set status:open project:$PROJECT topic:$TOPIC owner:$USERNAME) +previous=$(echo "$change_info" | grep "^ number:" | awk '{print $2}') +if [ -n "$previous" ]; then + change_id=$(echo "$change_info" | grep "^change" | awk '{print $2}') + # read return a non zero value when it reaches EOF. Because we use a + # heredoc here it will always reach EOF and return a nonzero value. + # Disable -e temporarily to get around the read. + # The reason we use read is to allow for multiline variable content + # and variable interpolation. Simply double quoting a string across + # multiple lines removes the newlines. + set +e + read -d '' COMMIT_MSG <