From 0e03bec4b1d7ecf76ab778f44f74759e076dc67e Mon Sep 17 00:00:00 2001 From: Skyler Berg Date: Tue, 1 Sep 2015 12:02:04 -0700 Subject: [PATCH] Add CI Watch, a third-party CI monitoring dashboard Original Change-Id: I8611f25a6700c6e0c64c3fadf820dbc9adcd5ea5 Change-Id: I847016f87ebd6da559ecd6298c5ad007bc935cb8 --- LICENSE | 176 ++++++++++++++++++++++++++ README.md | 53 ++++++++ ci-watch.conf.sample | 18 +++ ciwatch.wsgi | 17 +++ ciwatch/__init__.py | 31 +++++ ciwatch/api.py | 87 +++++++++++++ ciwatch/cache.py | 38 ++++++ ciwatch/config.py | 28 ++++ ciwatch/db.py | 57 +++++++++ ciwatch/events.py | 174 +++++++++++++++++++++++++ ciwatch/filters.py | 32 +++++ ciwatch/log.py | 46 +++++++ ciwatch/models.py | 98 ++++++++++++++ ciwatch/populate.py | 42 ++++++ ciwatch/static/hover.js | 23 ++++ ciwatch/static/style.css | 65 ++++++++++ ciwatch/static/verified.js | 33 +++++ ciwatch/templates/_contact.html.jinja | 1 + ciwatch/templates/_header.html.jinja | 3 + ciwatch/templates/_usage.html.jinja | 7 + ciwatch/templates/index.html.jinja | 111 ++++++++++++++++ ciwatch/templates/project.html.jinja | 109 ++++++++++++++++ ciwatch/views.py | 38 ++++++ requirements.txt | 4 + run.py | 18 +++ setup.py | 41 ++++++ 26 files changed, 1350 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 ci-watch.conf.sample create mode 100644 ciwatch.wsgi create mode 100644 ciwatch/__init__.py create mode 100644 ciwatch/api.py create mode 100644 ciwatch/cache.py create mode 100644 ciwatch/config.py create mode 100644 ciwatch/db.py create mode 100644 ciwatch/events.py create mode 100644 ciwatch/filters.py create mode 100644 ciwatch/log.py create mode 100644 ciwatch/models.py create mode 100644 ciwatch/populate.py create mode 100644 ciwatch/static/hover.js create mode 100644 ciwatch/static/style.css create mode 100644 ciwatch/static/verified.js create mode 100644 ciwatch/templates/_contact.html.jinja create mode 100644 ciwatch/templates/_header.html.jinja create mode 100644 ciwatch/templates/_usage.html.jinja create mode 100644 ciwatch/templates/index.html.jinja create mode 100644 ciwatch/templates/project.html.jinja create mode 100644 ciwatch/views.py create mode 100644 requirements.txt create mode 100644 run.py create mode 100644 setup.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..68c771a --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..f112236 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# CI Watch + +## Installation + +From this folder, run the following commands. + +``` +pip install -r requirements.txt +pip install -e . +``` + +These instructions are for development and testing installations. + +## Usage + +At the moment, this package provides three commands. + +`ci-watch-server`. +Launch a development server. + +`ci-watch-stream-events`. +Stream events from Gerrit and append valid events to `third-party-ci.log`. + +`ci-watch-populate-database`. +Add all entries from `third-party-ci.log` to the database. + + +## Configuration + +Configuration is stored in the `ci-watch.conf` file. Importantly, you can +specify a directory to store the `third-party-ci.log` file (data\_dir) as well +as the database to connect to. Look at `ci-watch.conf.sample` for an example. + +Other settings should be self explanatory based on the provided configuration +file. + +## State of the project + +This project is a work in progress and the code is pretty rough in some places. + +## TODO + +* Add tests. +* Use a different cache other than SimpleCache. It is not threadsafe. We + should use something like redis instead. + +These items are far from the only work needed for this project. + + +## Acknowledgements + +This code was originally forked from John Griffith's sos-ci project. Some of it +can still be found in the code and configuration file. diff --git a/ci-watch.conf.sample b/ci-watch.conf.sample new file mode 100644 index 0000000..5b94e88 --- /dev/null +++ b/ci-watch.conf.sample @@ -0,0 +1,18 @@ +[AccountInfo] +gerrit_ssh_key = /path/to/private/key +gerrit_username = your_username +gerrit_host = review.openstack.org +gerrit_port = 29418 + +[Data] +debug = True +data_dir = /var/data + +[database] +connection = sqlite:///:memory + +[misc] +; This is a more complete list that is not missing any CI's from the wiki +; It may include projects that don't actually do the third party CI thing +; projects = cinder,nova,swift,rally,murano,keystone,ironic,octavia,os-brick,neutron,tempest,neutron-lbaas,devstack,designate,manila +projects = cinder,nova,swift,rally,murano,ironic,octavia,os-brick,neutron,neutron-lbaas,devstack,manila diff --git a/ciwatch.wsgi b/ciwatch.wsgi new file mode 100644 index 0000000..7d9ae44 --- /dev/null +++ b/ciwatch.wsgi @@ -0,0 +1,17 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +sys.path.insert(0, "/var/www/ciwatch") +from ciwatch import app as application diff --git a/ciwatch/__init__.py b/ciwatch/__init__.py new file mode 100644 index 0000000..edecfb3 --- /dev/null +++ b/ciwatch/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from flask import Flask + +app = Flask(__name__) + +from ciwatch import views # noqa +from ciwatch import filters # noqa + + +__version__ = "0.0.1" + + +def main(): + app.run(debug=True) + + +if __name__ == '__main__': + main() diff --git a/ciwatch/api.py b/ciwatch/api.py new file mode 100644 index 0000000..64fdf62 --- /dev/null +++ b/ciwatch/api.py @@ -0,0 +1,87 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from collections import OrderedDict +from datetime import datetime, timedelta + +from flask import request +from sqlalchemy import and_ +from sqlalchemy import desc + +from ciwatch import db +from ciwatch.models import CiServer, Project, PatchSet + + +TIME_OPTIONS = OrderedDict([ # Map time options to hours + ("24 hours", 24), + ("48 hours", 48), + ("7 days", 7 * 24), +]) + +DEFAULT_TIME_OPTION = "24 hours" +DEFAULT_PROJECT = "cinder" + + +def _get_ci_info_for_patch_sets(ci, patch_sets): + ci_info = {"name": ci.name, "trusted": ci.trusted, "results": []} + for patch_set in patch_sets: + for comment in patch_set.comments: + if comment.ci_server_id == ci.id: + ci_info["results"].append(comment) + break + else: # nobreak + ci_info["results"].append(None) + return ci_info + + +def get_projects(): + return db.session.query(Project).order_by(Project.name).all() + + +def get_ci_servers(): + return db.session.query(CiServer).order_by( + desc(CiServer.trusted), CiServer.name).all() + + +def get_patch_sets(project, since): + return db.session.query(PatchSet).filter( + and_(PatchSet.project == project, PatchSet.created >= since) + ).order_by(PatchSet.created.desc()).all() + + +def get_time_options(): + return TIME_OPTIONS.keys() + + +def get_context(): + project = request.args.get('project', DEFAULT_PROJECT) + time = request.args.get('time', DEFAULT_TIME_OPTION) + since = datetime.now() - timedelta(hours=TIME_OPTIONS[time]) + project = db.session.query(Project).filter( + Project.name == project).one() + patch_sets = get_patch_sets(project=project, since=since) + results = OrderedDict() + for ci in get_ci_servers(): + ci_info = _get_ci_info_for_patch_sets(ci, patch_sets) + if any(result for result in ci_info["results"]): + results[ci.ci_owner] = results.get(ci.ci_owner, []) + results[ci.ci_owner].append( + _get_ci_info_for_patch_sets(ci, patch_sets)) + + return {"time_options": get_time_options(), + "time_option": time, + "patch_sets": patch_sets, + "project": project, + "projects": get_projects(), + "user_results": results} diff --git a/ciwatch/cache.py b/ciwatch/cache.py new file mode 100644 index 0000000..61526df --- /dev/null +++ b/ciwatch/cache.py @@ -0,0 +1,38 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from functools import wraps + +from flask import request +from werkzeug.contrib.cache import SimpleCache + + +cache = SimpleCache() + + +def cached(func): + @wraps(func) + def wrapper(*args, **kwargs): + key = _get_cache_key() + result = cache.get(key) + if result is None: + result = func(*args, **kwargs) + cache.set(key, result, timeout=60) + return result + return wrapper + + +def _get_cache_key(): + args = request.args + return request.path + str([(key, args[key]) for key in sorted(args)]) diff --git a/ciwatch/config.py b/ciwatch/config.py new file mode 100644 index 0000000..3d36e35 --- /dev/null +++ b/ciwatch/config.py @@ -0,0 +1,28 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from iniparse import INIConfig + +_fdir = os.path.dirname(os.path.realpath(__file__)) +_conf_dir = os.path.dirname(_fdir) +cfg = INIConfig(open(_conf_dir + '/ci-watch.conf')) + + +def get_projects(): + projects = [] + for name in cfg.misc.projects.split(','): + projects.append(name) + return projects diff --git a/ciwatch/db.py b/ciwatch/db.py new file mode 100644 index 0000000..f9c0b56 --- /dev/null +++ b/ciwatch/db.py @@ -0,0 +1,57 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from ciwatch import models +from ciwatch.config import cfg, get_projects + + +engine = create_engine(cfg.database.connection) +Session = sessionmaker() +Session.configure(bind=engine) +models.Base.metadata.create_all(engine) +session = Session() + + +def create_projects(): + for name in get_projects(): + get_or_create(models.Project, + commit_=False, + name=name) + session.commit() + + +def update_or_create_comment(commit_=True, **kwargs): + comment = session.query(models.Comment).filter_by( + ci_server_id=kwargs['ci_server_id'], + patch_set_id=kwargs['patch_set_id']).scalar() + if comment is not None: + for key, value in kwargs.iteritems(): + setattr(comment, key, value) + else: + session.add(models.Comment(**kwargs)) + if commit_: + session.commit() + + +def get_or_create(model, commit_=True, **kwargs): + result = session.query(model).filter_by(**kwargs).first() + if not result: + result = model(**kwargs) + session.add(result) + if commit_: + session.commit() + return result diff --git a/ciwatch/events.py b/ciwatch/events.py new file mode 100644 index 0000000..712e65c --- /dev/null +++ b/ciwatch/events.py @@ -0,0 +1,174 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import paramiko +import time +from datetime import datetime +import re + +from ciwatch import db, models +from ciwatch.log import logger, DATA_DIR +from ciwatch.config import cfg, get_projects + + +def _process_project_name(project_name): + return project_name.split('/')[-1] + + +def _process_event(event): + comment = event['comment'] + # Find all the CIs voting in this comment + lines = comment.splitlines() + event['ci-status'] = {} + for line in lines: + possible_results = "FAILURE|SUCCESS|NOT_REGISTERED|UNSTABLE" + pattern = re.compile("[-*]\s+([^\s*]+)\s+(http[^\s*]+) : (%s)" % + possible_results) + match = pattern.search(line) + if match is not None: + ci_name = match.group(1) + log_url = match.group(2) + result = match.group(3) + event['ci-status'][ci_name] = { + "result": result, + "log_url": log_url} + + +def _is_ci_user(name): + return 'CI' in name or 'Jenkins' in name + + +# Check if this is a third party CI event +def _is_valid(event): + if (event.get('type', 'nill') == 'comment-added' and + _is_ci_user(event['author'].get('name', '')) and + _process_project_name(event['change']['project']) in get_projects() and + event['change']['branch'] == 'master'): + return True + return False + + +def _store_event(event): + with open(DATA_DIR + '/third-party-ci.log', 'a') as f: + json.dump(event, f) + f.write('\n') + add_event_to_db(event) + return event + + +class GerritEventStream(object): + def __init__(self): + + logger.debug('Connecting to %(host)s:%(port)d as ' + '%(user)s using %(key)s', + {'user': cfg.AccountInfo.gerrit_username, + 'key': cfg.AccountInfo.gerrit_ssh_key, + 'host': cfg.AccountInfo.gerrit_host, + 'port': int(cfg.AccountInfo.gerrit_port)}) + + self.ssh = paramiko.SSHClient() + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + connected = False + while not connected: + try: + self.ssh.connect(cfg.AccountInfo.gerrit_host, + int(cfg.AccountInfo.gerrit_port), + cfg.AccountInfo.gerrit_username, + key_filename=cfg.AccountInfo.gerrit_ssh_key) + connected = True + except paramiko.SSHException as e: + logger.error('%s', e) + logger.warn('Gerrit may be down, will pause and retry...') + time.sleep(10) + + self.stdin, self.stdout, self.stderr =\ + self.ssh.exec_command("gerrit stream-events") + + def __iter__(self): + return self + + def next(self): + return self.stdout.readline() + + +def parse_json_event(event): + try: + event = json.loads(event) + except Exception as ex: + logger.error('Failed json.loads on event: %s', event) + logger.exception(ex) + return None + if _is_valid(event): + _process_event(event) + logger.info('Parsed valid event: %s', event) + return event + return None + + +def add_event_to_db(event, commit_=True): + project = db.session.query(models.Project).filter( + models.Project.name == _process_project_name( + event["change"]["project"])).one() + patch_set = db.get_or_create( + models.PatchSet, + commit_=False, + project_id=project.id, + ref=event['patchSet']['ref'], + commit_message=event['change']['commitMessage'], + created=datetime.fromtimestamp( + int(event['patchSet']['createdOn']))) + + owner_name = event["author"]["name"] + owner = db.get_or_create(models.CiOwner, name=owner_name) + trusted = (event["author"]["username"] == "jenkins") + + if trusted and "approvals" in event: + if event["approvals"][0]["value"] in ("+1", "+2"): + patch_set.verified = True + elif event["approvals"][0]["value"] in ("-1", "-2"): + patch_set.verified = False + + for ci, data in event['ci-status'].iteritems(): + ci_server = db.get_or_create(models.CiServer, + commit_=False, + name=ci, + trusted=trusted, + ci_owner_id=owner.id) + db.update_or_create_comment(commit_=False, + result=data["result"], + log_url=data["log_url"], + ci_server_id=ci_server.id, + patch_set_id=patch_set.id) + if commit_: + db.session.commit() + + +def main(): + db.create_projects() # This will make sure the database has projects in it + while True: + try: + events = GerritEventStream() + except paramiko.SSHException as ex: + logger.exception('Error connecting to Gerrit: %s', ex) + time.sleep(60) + for event in events: + event = parse_json_event(event) + if event is not None: + _store_event(event) + + +if __name__ == '__main__': + main() diff --git a/ciwatch/filters.py b/ciwatch/filters.py new file mode 100644 index 0000000..088e895 --- /dev/null +++ b/ciwatch/filters.py @@ -0,0 +1,32 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re + +from jinja2 import evalcontextfilter, Markup, escape + +from ciwatch import app + + +_paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}') + + +@app.template_filter() +@evalcontextfilter +def nl2br(eval_ctx, value): + result = u'\n\n'.join(u'

%s

' % p.replace('\n', '
\n') + for p in _paragraph_re.split(escape(value))) + if eval_ctx.autoescape: + result = Markup(result) + return result diff --git a/ciwatch/log.py b/ciwatch/log.py new file mode 100644 index 0000000..1f887df --- /dev/null +++ b/ciwatch/log.py @@ -0,0 +1,46 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +from logging import handlers +import os + +from ciwatch.config import cfg + + +def setup_logger(name): + + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + log_formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]" + " %(message)s") + + file_handler =\ + handlers.RotatingFileHandler(name, + maxBytes=1048576, + backupCount=2,) + logger.addHandler(file_handler) + + console_handler = logging.StreamHandler() + console_handler.setFormatter(log_formatter) + logger.addHandler(console_handler) + return logger + + +DATA_DIR =\ + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/data' +if cfg.Data.data_dir: + DATA_DIR = cfg.Data.data_dir + +logger = setup_logger(DATA_DIR + '/ci-watch.log') diff --git a/ciwatch/models.py b/ciwatch/models.py new file mode 100644 index 0000000..cc05cbc --- /dev/null +++ b/ciwatch/models.py @@ -0,0 +1,98 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Boolean, Column, DateTime, Integer, String, ForeignKey +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship, backref + + +Base = declarative_base() + + +class Project(Base): + __tablename__ = "projects" + + id = Column(Integer, primary_key=True) + name = Column(String, unique=True) + + def __repr__(self): + return "" % self.name + + +class PatchSet(Base): + __tablename__ = "patch_sets" + + id = Column(Integer, primary_key=True) + created = Column(DateTime) + # ref = Column(String, unique=True) + ref = Column(String) # Why are there duplicate refs? + # Verified only represents Jenkin's vote + verified = Column(Boolean, nullable=True, default=None) + + commit_message = Column(String) + + project_id = Column(Integer, ForeignKey('projects.id')) + project = relationship("Project", backref=backref('patch_sets', + order_by=id)) + + def __repr__(self): + return "" % ( + self.created, self.ref) + + +class Comment(Base): + __tablename__ = "comments" + + id = Column(Integer, primary_key=True) + result = Column(String) + log_url = Column(String, nullable=True, default=None) + + ci_server_id = Column(Integer, ForeignKey('ci_servers.id')) + ci_server = relationship("CiServer", backref=backref('comments', + order_by=id)) + + patch_set_id = Column(Integer, ForeignKey('patch_sets.id')) + patch_set = relationship("PatchSet", backref=backref('comments', + order_by=id)) + + def __repr__(self): + return "" % ( + self.log_url, self.result) + + +class CiServer(Base): + __tablename__ = "ci_servers" + + id = Column(Integer, primary_key=True) + name = Column(String) + + # Official OpenStack CIs are trusted (e.g., Jenkins) + trusted = Column(Boolean, default=False) + + ci_owner_id = Column(Integer, ForeignKey('ci_owners.id')) + ci_owner = relationship('CiOwner', backref=backref('ci_servers', + order_by=id)) + + def __repr__(self): + return "" % self.name + + +class CiOwner(Base): + __tablename__ = "ci_owners" + + id = Column(Integer, primary_key=True) + name = Column(String, unique=True) + + def __repr__(self): + return "" % self.name diff --git a/ciwatch/populate.py b/ciwatch/populate.py new file mode 100644 index 0000000..028baa2 --- /dev/null +++ b/ciwatch/populate.py @@ -0,0 +1,42 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ciwatch import db +from ciwatch.events import parse_json_event, add_event_to_db + + +def get_data(): + data = [] + with open('/var/data/third-party-ci.log') as file_: + for line in file_: + event = parse_json_event(line) + if event is not None: + data.append(event) + return data + + +def load_data(): + data = get_data() + for event in data: + add_event_to_db(event, commit_=False) + db.session.commit() + + +def main(): + db.create_projects() + load_data() + + +if __name__ == '__main__': + main() diff --git a/ciwatch/static/hover.js b/ciwatch/static/hover.js new file mode 100644 index 0000000..cbedd77 --- /dev/null +++ b/ciwatch/static/hover.js @@ -0,0 +1,23 @@ +// Code adapted from https://css-tricks.com/row-and-column-highlighting/ +$(document).ready(function () { + $("table").delegate("td.result","mouseover mouseleave", function(e) { + if (e.type == "mouseover") { + $(this).parent().addClass("hover"); + $("colgroup").eq($(this).index()).addClass("hover"); + } + else { + $(this).parent().removeClass("hover"); + $("colgroup").eq($(this).index()).removeClass("hover"); + } + }); + $("table").delegate("td.ci-name","mouseover mouseleave", function(e) { + if (e.type == "mouseover") { + $(this).parent().addClass("hover"); + } + else { + $(this).parent().removeClass("hover"); + } + }); + + $('[data-toggle="popover"]').popover({trigger: 'hover'}); +}); diff --git a/ciwatch/static/style.css b/ciwatch/static/style.css new file mode 100644 index 0000000..42a9461 --- /dev/null +++ b/ciwatch/static/style.css @@ -0,0 +1,65 @@ +table.table { + white-space: nowrap; +} + +.hover { + background-color: #EEE; +} + +.failure { + color: #C00; + font-weight: bold; +} + +.success { + color: #0C0; + font-weight: bold; +} + +.unstable { + color: #BB0; + font-weight: bold; +} + +.unregistered { + color: #333; + font-weight: bold; +} + +.ci-user { + background-color: #DDD !important; + font-style: italic; +} + +.blank-row { + height: 20px; +} + +.popover { + max-width: 100%; +} + +.no-style-link:link { + text-decoration: none; + color: #333; +} + +.no-style-link:visited { + text-decoration: none; + color: #333; +} + +.no-style-link:hover { + text-decoration: none; + color: #333; +} + +.no-style-link:active { + text-decoration: none; + color: #333; +} + +.glyphicon-none:before { + content: "\2122"; + color: transparent !important; +} diff --git a/ciwatch/static/verified.js b/ciwatch/static/verified.js new file mode 100644 index 0000000..ee3eaf9 --- /dev/null +++ b/ciwatch/static/verified.js @@ -0,0 +1,33 @@ +/* Copyright (c) 2015 Tintri. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +var toggle1 = function () { + $(".verified-1").css("background-color", "#FAA"); + $(this).one("click", toggle2); +} + +var toggle2 = function () { + $(".verified-1").css("background-color", ""); + $(this).one("click", toggle1); +} + +$(document).ready(function () { + $("colgroup").each(function (i, elem) { + if ($(elem).hasClass("verified-1")) { + $("#results").find("td").filter(":nth-child(" + (i + 1) + ")").addClass("verified-1"); + } + }); + $("#verified-1-button").one("click", toggle1); +}); diff --git a/ciwatch/templates/_contact.html.jinja b/ciwatch/templates/_contact.html.jinja new file mode 100644 index 0000000..e3c477c --- /dev/null +++ b/ciwatch/templates/_contact.html.jinja @@ -0,0 +1 @@ +

Send feedback to openstack-dev.tintri.com

diff --git a/ciwatch/templates/_header.html.jinja b/ciwatch/templates/_header.html.jinja new file mode 100644 index 0000000..e40d27d --- /dev/null +++ b/ciwatch/templates/_header.html.jinja @@ -0,0 +1,3 @@ +
+

CI Watch — an OpenStack third-party CI monitoring tool

+
diff --git a/ciwatch/templates/_usage.html.jinja b/ciwatch/templates/_usage.html.jinja new file mode 100644 index 0000000..ea95e55 --- /dev/null +++ b/ciwatch/templates/_usage.html.jinja @@ -0,0 +1,7 @@ +

+ Each project has a table of recent CI results. + Columns plot patch sets and rows plot CI servers. + Each cell shows results for the row's CI server and the column's patch set. + Click the icons to view relevant logs. + Newest results are shown furthest to the left. +

diff --git a/ciwatch/templates/index.html.jinja b/ciwatch/templates/index.html.jinja new file mode 100644 index 0000000..3b7a137 --- /dev/null +++ b/ciwatch/templates/index.html.jinja @@ -0,0 +1,111 @@ + + + + + + + + + + + +
+ + {% include '_header.html.jinja' %} + +
+

Select a project to view CI results

+
+
+ {% for project in projects %} +
+ +

{{ project.name }}

+ +
+ {% endfor %} +
+
+ +

Usage

+ {% include '_usage.html.jinja' %} + +
+
Legend
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IconMeaning
+ + + + Link to patch set on Gerrit
+

+ + +

+ 		CI voted SUCCESS for patch set

+

+ + +

+ 		CI voted FAILURE for patch set
+

+ + +

+ 		CI voted UNSTABLE for patch set
+

N

+ 		CI voted NOT_REGISTERED patch set
+ + + CI has not yet voted on this patch set
+
+ +

Contact Us

+ {% include '_contact.html.jinja' %} + +
+ + + + + + diff --git a/ciwatch/templates/project.html.jinja b/ciwatch/templates/project.html.jinja new file mode 100644 index 0000000..911b6c2 --- /dev/null +++ b/ciwatch/templates/project.html.jinja @@ -0,0 +1,109 @@ + + + + + + + + + + +
+ {% include '_header.html.jinja' %} + {% include '_usage.html.jinja' %} + {% include '_contact.html.jinja' %} +
Viewing results for {{ project.name }} from the past {{ request.args.get('time', time_option) }}.
+
+ +
+
+ + +
+ +
+ +
    + {% for option in time_options %} +
  • {{ option }}
    • + {% endfor %} +
+
+ + +
+ +
+ +
+ + + + {% for patch_set in patch_sets %} + {% if patch_set.verified is not none and not patch_set.verfied %} + + {% else %} + + {% endif %} + {% endfor %} + + + + + + {% for patch_set in patch_sets %} + + {% endfor %} + + + {% for owner, results in user_results.iteritems() %} + + {% for ci in results %} + + + {% for comment in ci["results"] %} + + {% endfor %} + + {% endfor %} + + {% endfor %} + +
Patch Set + + + +
{{ owner.name }}
{{ ci["name"] }} + {% if comment is not none %} + + {% if comment.result == "SUCCESS" %} +

+ {% elif comment.result == "FAILURE" %} +

+ {% elif comment.result == "UNSTABLE" %} +

+ {% elif comment.result == "NOT_REGISTERED" %} +

N

+ {% endif %} + + {% endif %} +
+
+ + + + + + + + diff --git a/ciwatch/views.py b/ciwatch/views.py new file mode 100644 index 0000000..31a5430 --- /dev/null +++ b/ciwatch/views.py @@ -0,0 +1,38 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from flask import render_template +from sqlalchemy.orm.exc import NoResultFound +from werkzeug.exceptions import abort + +from ciwatch import app +from ciwatch.api import get_context +from ciwatch.api import get_projects +from ciwatch.cache import cached + + +@app.route("/") +@app.route("/index") +@app.route("/home") +def home(): + return render_template('index.html.jinja', projects=get_projects()) + + +@app.route("/project") +@cached +def project(): + try: + return render_template("project.html.jinja", **get_context()) + except NoResultFound: + abort(404) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..157ff9a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +flask +sqlalchemy +iniparse +paramiko diff --git a/run.py b/run.py new file mode 100644 index 0000000..f00d35d --- /dev/null +++ b/run.py @@ -0,0 +1,18 @@ +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ciwatch + +if __name__ == "__main__": + ciwatch.app.run(debug=True) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..84ae731 --- /dev/null +++ b/setup.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 Tintri. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from os.path import join, dirname + +from setuptools import setup + +import ciwatch + + +setup( + name='ci-watch', + version=ciwatch.__version__, + long_description=open(join(dirname(__file__), 'README.md')).read(), + entry_points={ + 'console_scripts': [ + 'ci-watch-server = ciwatch:main', + 'ci-watch-populate-database = ciwatch.populate:main', + 'ci-watch-stream-events = ciwatch.events:main', + ], + }, + install_requires=[ + "flask", + "sqlalchemy", + "iniparse", + "paramiko", + ] +)