diff --git a/export_static.py b/export_static.py index 250a9c1..239f36d 100644 --- a/export_static.py +++ b/export_static.py @@ -12,16 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. -import os -import gzip -import shutil +from __future__ import print_function + import django +import gzip +import os +import shutil from argparse import ArgumentParser from django.http import Http404 -from django.test import RequestFactory from django.core.urlresolvers import resolve +from django.test import RequestFactory from stackviz.parser import tempest_subunit from stackviz import settings @@ -62,7 +64,7 @@ def export_single_page(path, dest_dir, use_gzip=False): with open_func(os.path.join(dest_dir, dest_file), 'wb') as f: f.write(content) except Http404 as ex: - print "Warning: skipping %s due to error: %s" % (path, ex.message) + print("Warning: skipping %s due to error: %s" % (path, ex.message)) def init_django(args): @@ -105,35 +107,35 @@ def main(): if not args.ignore_bower: if not os.listdir(os.path.join('stackviz', 'static', 'components')): - print "Bower components have not been installed, please run " \ - "`bower install`" + print("Bower components have not been installed, please run " + "`bower install`") return 1 if os.path.exists(args.path): if os.listdir(args.path): - print "Destination exists and is not empty, cannot continue" + print("Destination exists and is not empty, cannot continue") return 1 os.mkdir(args.path) init_django(args) - print "Copying static files ..." + print("Copying static files ...") shutil.copytree(os.path.join('stackviz', 'static'), os.path.join(args.path, 'static')) for path in EXPORT_PATHS: - print "Rendering:", path + print("Rendering:", path) export_single_page(path, args.path) repos = tempest_subunit.get_repositories() if repos: for run_id in range(repos[0].count()): - print "Rendering views for tempest run #%d" % run_id + print("Rendering views for tempest run #%d" % (run_id)) export_single_page('/tempest_timeline_%d.html' % run_id, args.path) export_single_page('/tempest_results_%d.html' % run_id, args.path) - print "Exporting data for tempest run #%d" % run_id + print("Exporting data for tempest run #%d" % (run_id)) export_single_page('/tempest_api_tree_%d.json' % run_id, args.path, args.gzip) export_single_page('/tempest_api_raw_%d.json' % run_id, @@ -141,10 +143,10 @@ def main(): export_single_page('/tempest_api_details_%d.json' % run_id, args.path, args.gzip) else: - print "Warning: no test repository could be loaded, no data will be " \ - "available!" + print("Warning: no test repository could be loaded, no data will " + "be available!") - print "Exporting DStat log: dstat_log.csv" + print("Exporting DStat log: dstat_log.csv") export_single_page('/dstat_log.csv', args.path, args.gzip) diff --git a/stackviz/global_template_injector.py b/stackviz/global_template_injector.py index b0b9fd8..7b0cef9 100644 --- a/stackviz/global_template_injector.py +++ b/stackviz/global_template_injector.py @@ -15,6 +15,7 @@ from stackviz.parser.tempest_subunit import get_repositories from stackviz.settings import USE_GZIP + def inject_extra_context(request): ret = { 'use_gzip': USE_GZIP diff --git a/stackviz/parser/devstack_parser.py b/stackviz/parser/devstack_parser.py index ece088f..bd98e4f 100644 --- a/stackviz/parser/devstack_parser.py +++ b/stackviz/parser/devstack_parser.py @@ -33,7 +33,8 @@ import os -from datetime import datetime, timedelta +from datetime import datetime +from datetime import timedelta from log_node import LogNode #: The format of the timestamp prefixing each log entry @@ -41,6 +42,7 @@ TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S.%f' def extract_date(line): + """ Extracts a date from the given line, returning the parsed date and remaining contents of the line. @@ -48,6 +50,7 @@ def extract_date(line): :param line: the line to extract a date from :return: a tuple of the parsed date and remaining line contents """ + date_str, message = line.split(' | ', 1) date = datetime.strptime(date_str, TIMESTAMP_FORMAT) @@ -55,6 +58,7 @@ def extract_date(line): def parse_summary(summary_path): + """ Parses a summary logfile. Summary entries are prefixed with identical datestamps to those in the main log, but have only explicit log messages @@ -66,6 +70,7 @@ def parse_summary(summary_path): :param summary_path: the path to the summary file to parse :return: a list of ordered `LogNode` instances """ + ret = [] last_node = None @@ -85,6 +90,7 @@ def parse_summary(summary_path): def parse_log(log_path): + """ Parses a general `stack.sh` logfile, forming a full log tree based on the hierarchy of nested commands as presented in the log. @@ -96,6 +102,7 @@ def parse_log(log_path): :param log_path: the path to the logfile to parse :return: a list of parsed `LogNode` instances """ + last_depth = 1 last_node = None @@ -144,6 +151,7 @@ def parse_log(log_path): def merge(summary, log): + """ Merges general log entries into parent categories based on their timestamp relative to the summary output timestamp. @@ -157,6 +165,7 @@ def merge(summary, log): :param log: the list of general log nodes :return: the original summary nodes with children set to the log nodes """ + if not summary: return [] @@ -180,6 +189,7 @@ def merge(summary, log): def bootstrap(log_path, summary_path=None): + """ Loads, parses, and merges the given log and summary files. The path to the summary file will be determined automatically based on the path to the @@ -193,6 +203,7 @@ def bootstrap(log_path, summary_path=None): :return: a list of merged `LogNode` instances, or `None` if no matching summary file can be located automatically """ + if summary_path: return merge(parse_summary(summary_path), parse_log(log_path)) @@ -244,4 +255,3 @@ def get_command_totals(node, totals=None): totals[combined] += entry.duration_self return totals - diff --git a/stackviz/parser/log_node.py b/stackviz/parser/log_node.py index 4da0b8f..aa60e78 100644 --- a/stackviz/parser/log_node.py +++ b/stackviz/parser/log_node.py @@ -12,15 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. +from datetime import datetime +from datetime import timedelta + from inspect import getmembers from numbers import Number -from datetime import datetime, timedelta #: The default cutoff for log entries when pruning takes place, in seconds DEFAULT_PRUNE_CUTOFF = 0.05 -class LogNode: +class LogNode(object): + """ Represents an entry in an ordered event log, consisting of a date, message, and an arbitrary set of child nodes. @@ -41,10 +44,12 @@ class LogNode: @property def duration(self): + """ Determines the overall duration for this node, beginning at this parent node's start time through the final child's ending time. """ + if self.children: last_sibling = self.children[-1].next_sibling if not last_sibling: @@ -69,6 +74,7 @@ class LogNode: return self.next_sibling.date - self.date def traverse(self): + """ A generator that will traverse all child nodes of this log tree sequentially. @@ -155,17 +161,18 @@ class LogNode: def prune(nodes, cutoff=DEFAULT_PRUNE_CUTOFF, fill=None): + """ Prunes the given list of `LogNode` instances, removing nodes whose duration is less than the given cutoff value. If a `fill` value is provided, removed nodes will be replaced with a single filler value accounting for the lost - duration. This filler value will be inserted at the end of the list and will - not be properly linked to other values. + duration. This filler value will be inserted at the end of the list and + will not be properly linked to other values. Note that returned values will not necessarily be a continuous list of - nodes. The original list will remain unchanged; sibling and child references - will not be modified to point to account any modified, removed, or added - nodes. + nodes. The original list will remain unchanged; sibling and child + references will not be modified to point to account any modified, removed, + or added nodes. :param nodes: the list of log nodes to prune :type nodes: list[LogNode] @@ -176,6 +183,7 @@ def prune(nodes, cutoff=DEFAULT_PRUNE_CUTOFF, fill=None): :type cutoff: float :return: a (potentially) reduced list of nodes """ + ret = [] fill_amount = 0.0 diff --git a/stackviz/parser/tempest_subunit.py b/stackviz/parser/tempest_subunit.py index e4e59e6..dfa79e3 100644 --- a/stackviz/parser/tempest_subunit.py +++ b/stackviz/parser/tempest_subunit.py @@ -17,13 +17,13 @@ import re from functools import partial from subunit import ByteStreamToStreamResult -from testtools import (StreamResult, StreamSummary, - StreamToDict, CopyStreamResult) +from testtools import CopyStreamResult +from testtools import StreamResult +from testtools import StreamSummary +from testtools import StreamToDict -from testrepository.repository import AbstractTestRun -from testrepository.repository.file import (RepositoryFactory, - Repository, - RepositoryNotFound) +from testrepository.repository.file import RepositoryFactory +from testrepository.repository.file import RepositoryNotFound from stackviz import settings @@ -33,6 +33,7 @@ NAME_TAGS_PATTERN = re.compile(r'^(.+)\[(.+)\]$') def get_repositories(): + """ Loads all test repositories from locations configured in `settings.TEST_REPOSITORIES`. Only locations with a valid `.testrepository` @@ -41,6 +42,7 @@ def get_repositories(): :return: a list of loaded :class:`Repository` instances :rtype: list[Repository] """ + factory = RepositoryFactory() ret = [] @@ -48,7 +50,7 @@ def get_repositories(): for path in settings.TEST_REPOSITORIES: try: ret.append(factory.open(path)) - except (ValueError, RepositoryNotFound) as ex: + except (ValueError, RepositoryNotFound): # skip continue @@ -56,7 +58,7 @@ def get_repositories(): def _clean_name(name): - # TODO: currently throwing away other info - any worth keeping? + # TODO(currently throwing away other info - any worth keeping?) m = NAME_TAGS_PATTERN.match(name) if m: # tags = m.group(2).split(',') @@ -94,16 +96,18 @@ def _read_test(test, out, strip_details): def convert_run(test_run, strip_details=False): + """ - Converts the given test run into a raw list of test dicts, using the subunit - stream as an intermediate format. + Converts the given test run into a raw list of test dicts, using the + subunit stream as an intermediate format.(see: read_subunit.py from + subunit2sql) :param test_run: the test run to convert :type test_run: AbstractTestRun :param strip_details: if True, remove test details (e.g. stdout/stderr) :return: a list of individual test results """ - # see: read_subunit.py from subunit2sql + ret = [] stream = test_run.get_subunit_stream() @@ -150,6 +154,7 @@ def _descend_recurse(parent, parts_remaining): def _descend(root, path): + """ Retrieves the node within the `root` dict denoted by the series of '.'-separated children as specified in `path`. Children for each node must @@ -164,6 +169,7 @@ def _descend(root, path): :type path: str :return: the dict node representing the last child """ + path_parts = path.split('.') path_parts.reverse() @@ -173,6 +179,7 @@ def _descend(root, path): def reorganize(converted_test_run): + """ Reorganizes and categorizes the given test run, forming tree of tests categorized by their module paths. @@ -180,6 +187,7 @@ def reorganize(converted_test_run): :param converted_test_run: :return: a dict tree of test nodes, organized by module path """ + ret = {} for entry in converted_test_run: diff --git a/stackviz/urls.py b/stackviz/urls.py index 7eb46b8..fd6af7f 100644 --- a/stackviz/urls.py +++ b/stackviz/urls.py @@ -12,8 +12,9 @@ # License for the specific language governing permissions and limitations # under the License. -from django.conf.urls import patterns, include, url -from django.contrib import admin +from django.conf.urls import include +from django.conf.urls import patterns +from django.conf.urls import url from stackviz.views.index import IndexView @@ -23,6 +24,6 @@ urlpatterns = patterns( url(r'^index.html$', IndexView.as_view(), name="index"), url(r'^tempest_', include('stackviz.views.tempest.urls')), url(r'^devstack_', include('stackviz.views.devstack.urls')), - url(r'^upstream_', include ('stackviz.views.upstream.urls')), - url(r'^dstat_', include ('stackviz.views.dstat.urls')) + url(r'^upstream_', include('stackviz.views.upstream.urls')), + url(r'^dstat_', include('stackviz.views.dstat.urls')) ) diff --git a/stackviz/views/devstack/results.py b/stackviz/views/devstack/results.py index c8c498f..ca9471e 100644 --- a/stackviz/views/devstack/results.py +++ b/stackviz/views/devstack/results.py @@ -14,5 +14,6 @@ from django.views.generic import TemplateView + class ResultsView(TemplateView): template_name = 'devstack/results.html' diff --git a/stackviz/views/devstack/urls.py b/stackviz/views/devstack/urls.py index 228873e..059445f 100644 --- a/stackviz/views/devstack/urls.py +++ b/stackviz/views/devstack/urls.py @@ -12,16 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -from django.conf.urls import patterns, include, url -from django.contrib import admin +from django.conf.urls import patterns +from django.conf.urls import url from stackviz.views.devstack.results import ResultsView urlpatterns = patterns('', - # Examples: - # url(r'^$', 'stackviz.views.home', name='home'), - # url(r'^blog/', include('blog.urls')), - - url(r'^results$', ResultsView.as_view()), -) + url(r'^results$', ResultsView.as_view()), + ) diff --git a/stackviz/views/dstat/api.py b/stackviz/views/dstat/api.py index b95ab9d..ab6754f 100644 --- a/stackviz/views/dstat/api.py +++ b/stackviz/views/dstat/api.py @@ -12,9 +12,9 @@ # License for the specific language governing permissions and limitations # under the License. -import os +from django.http import Http404 +from django.http import HttpResponse -from django.http import HttpResponse, Http404 from django.views.generic import View from stackviz import settings diff --git a/stackviz/views/dstat/urls.py b/stackviz/views/dstat/urls.py index b887637..3c83633 100644 --- a/stackviz/views/dstat/urls.py +++ b/stackviz/views/dstat/urls.py @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -from django.conf.urls import patterns, include, url +from django.conf.urls import patterns +from django.conf.urls import url -from .api import DStatCSVEndpoint +from api import DStatCSVEndpoint -urlpatterns = patterns('', - url(r'^log.csv$', DStatCSVEndpoint.as_view()), -) + +urlpatterns = patterns('', url(r'^log.csv$', DStatCSVEndpoint.as_view())) diff --git a/stackviz/views/index.py b/stackviz/views/index.py index 658693a..316ebe5 100644 --- a/stackviz/views/index.py +++ b/stackviz/views/index.py @@ -14,5 +14,6 @@ from django.views.generic import TemplateView + class IndexView(TemplateView): template_name = 'index.html' diff --git a/stackviz/views/tempest/aggregate.py b/stackviz/views/tempest/aggregate.py index 57325e1..b7d1d59 100644 --- a/stackviz/views/tempest/aggregate.py +++ b/stackviz/views/tempest/aggregate.py @@ -14,5 +14,6 @@ from django.views.generic import TemplateView + class AggregateResultsView(TemplateView): - template_name = 'tempest/aggregate.html' \ No newline at end of file + template_name = 'tempest/aggregate.html' diff --git a/stackviz/views/tempest/api.py b/stackviz/views/tempest/api.py index e9a309a..6e7e8cc 100644 --- a/stackviz/views/tempest/api.py +++ b/stackviz/views/tempest/api.py @@ -15,9 +15,9 @@ from django.http import Http404 from restless.views import Endpoint -from stackviz.parser.tempest_subunit import (get_repositories, - convert_run, - reorganize) +from stackviz.parser.tempest_subunit import convert_run +from stackviz.parser.tempest_subunit import get_repositories +from stackviz.parser.tempest_subunit import reorganize #: Cached results from loaded subunit logs indexed by their run number _cached_run = {} @@ -33,9 +33,11 @@ _cached_details = {} class NoRunDataException(Http404): pass + class RunNotFoundException(Http404): pass + class TestNotFoundException(Http404): pass @@ -53,7 +55,7 @@ def _load_run(run_id): run = repos[0].get_test_run(run_id) # strip details for now - # TODO: provide method for getting details on demand + # TODO(provide method for getting details on demand) # (preferably for individual tests to avoid bloat) converted_run = convert_run(run, strip_details=True) _cached_run[run_id] = converted_run @@ -118,4 +120,3 @@ class TempestRunTreeEndpoint(Endpoint): class TempestRunDetailsEndpoint(Endpoint): def get(self, request, run_id, test_name=None): return _load_details(run_id, test_name) - diff --git a/stackviz/views/tempest/results.py b/stackviz/views/tempest/results.py index 8bc2f43..f206d3d 100644 --- a/stackviz/views/tempest/results.py +++ b/stackviz/views/tempest/results.py @@ -12,11 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -from django.core.urlresolvers import reverse -from django.views.generic import TemplateView, RedirectView -from django.http import Http404 - -from stackviz.parser.tempest_subunit import get_repositories +from django.views.generic import TemplateView class ResultsView(TemplateView): @@ -27,4 +23,3 @@ class ResultsView(TemplateView): context['run_id'] = self.kwargs['run_id'] return context - diff --git a/stackviz/views/tempest/timeline.py b/stackviz/views/tempest/timeline.py index 9516a75..8a339b8 100644 --- a/stackviz/views/tempest/timeline.py +++ b/stackviz/views/tempest/timeline.py @@ -12,11 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -from django.core.urlresolvers import reverse -from django.views.generic import TemplateView, RedirectView -from django.http import Http404 - -from stackviz.parser.tempest_subunit import get_repositories +from django.views.generic import TemplateView class TimelineView(TemplateView): @@ -27,4 +23,3 @@ class TimelineView(TemplateView): context['run_id'] = self.kwargs['run_id'] return context - diff --git a/stackviz/views/tempest/urls.py b/stackviz/views/tempest/urls.py index 2504e17..21e5fdd 100644 --- a/stackviz/views/tempest/urls.py +++ b/stackviz/views/tempest/urls.py @@ -12,15 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. -from django.conf.urls import patterns, include, url -from .results import ResultsView -from .timeline import TimelineView -from .aggregate import AggregateResultsView +from django.conf.urls import patterns +from django.conf.urls import url -from .api import (TempestRunTreeEndpoint, - TempestRunRawEndpoint, - TempestRunDetailsEndpoint) +from aggregate import AggregateResultsView +from results import ResultsView +from timeline import TimelineView + +from api import TempestRunDetailsEndpoint +from api import TempestRunRawEndpoint +from api import TempestRunTreeEndpoint urlpatterns = patterns('', diff --git a/stackviz/views/upstream/api.py b/stackviz/views/upstream/api.py index 698f3fe..61d1ece 100644 --- a/stackviz/views/upstream/api.py +++ b/stackviz/views/upstream/api.py @@ -12,30 +12,34 @@ # License for the specific language governing permissions and limitations # under the License. -from subunit2sql.db import api, models -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.orm.state import InstanceState from restless.views import Endpoint -import json +from subunit2sql.db import api + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + def _get_runs(change_id): + ''' When given the change_id of a Gerrit change, a connection will be made to the upstream subunit2sql db and query all run meta having that change_id :param change_id: the Gerrit change_id to query :return: a json dict of run_meta objects ''' - engine=create_engine('mysql://query:query@logstash.openstack.org:3306/subunit2sql') + + engine = create_engine('mysql://query:query@logstash.openstack.org' + + ':3306/subunit2sql') Session = sessionmaker(bind=engine) # create a Session session = Session() - list_of_runs = api.get_runs_by_key_value(key="build_change",value=change_id, - session=session) + list_of_runs = api.get_runs_by_key_value(key="build_change", + value=change_id, + session=session) ret_list = [] for run in list_of_runs: @@ -47,10 +51,12 @@ def _get_runs(change_id): class GerritURLEndpoint(Endpoint): def get(self, request, change_id): + ''' :param request: :param change_id: :return: Collection of run objects associated with a specific CID ''' + return _get_runs(change_id) diff --git a/stackviz/views/upstream/run.py b/stackviz/views/upstream/run.py index 058fbd2..75d36ef 100644 --- a/stackviz/views/upstream/run.py +++ b/stackviz/views/upstream/run.py @@ -14,17 +14,6 @@ from django.views.generic import TemplateView -# TODO Planned f(x): -# 1. Input a run_id from an upstream run to display run info -# 2. Compare runs by metadata -# -# 1. -# EX: url for logstash= -# http://logs.openstack.org/92/206192/2/check/gate-subunit2sql-python27/c1ff374/ -# -# a. link between logstash and subunit2sql (urlparser) -# b. display server-side as well as client-side logs -# + class RunView(TemplateView): template_name = 'upstream/run.html' - diff --git a/stackviz/views/upstream/test.py b/stackviz/views/upstream/test.py index 969bfec..133f93a 100644 --- a/stackviz/views/upstream/test.py +++ b/stackviz/views/upstream/test.py @@ -14,9 +14,10 @@ from django.views.generic import TemplateView -# TODO: Planned f(x): +# TODO(Planned functionality) # Compare one specific test against its moving average # + class TestView(TemplateView): - template_name = 'upstream/test.html' \ No newline at end of file + template_name = 'upstream/test.html' diff --git a/stackviz/views/upstream/urls.py b/stackviz/views/upstream/urls.py index a7aa26e..b6c09a2 100644 --- a/stackviz/views/upstream/urls.py +++ b/stackviz/views/upstream/urls.py @@ -12,12 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -from django.conf.urls import patterns, include, url +from django.conf.urls import patterns +from django.conf.urls import url -from .run import RunView -from .test import TestView +from run import RunView +from test import TestView -from .api import GerritURLEndpoint +from api import GerritURLEndpoint urlpatterns = patterns('',