Implementation of blueprint stackalytics-core

* Data updater is implemented
* Completed implementation of commit processor
* Logging is added into commit processor and runtime storage
* Commit processor is fixed
* Domain-company map is inverted
* Extracted get update count into separate function
* Fixed regex that matches diff statistics (lines inserted, lines deleted and files changed)
* Implemented caching of unknown users
* Replaced dictionaries by sets for pids and branches
* Vcs is responsible for module and branches fields of commit record
* Added release tags support
* Implemented statistics by company
* Added config for releases
* Implemented front-end for companies details
* Implemented front-end for modules details
* Fixed metric switch
* Implemented timeline rendering
* Release selector is fixed
* Chdir is needed after cloning a new repo
* Company details screen is implemented
* Fixed invalid emails processing by Launchpad
* Fixed parsing of 0 files changed case
* Module details screen implemented
* Commit message is cleared and links are inserted
* Engineer details screen is implemented
* Fixed mapping from company to email for subdomains of 3rd level
* Fixed wrong user structure for users not found by LP
* Also coverage for commit processor
* Fixed company matching algorithm
* The company was not matched when user email had more domains than company's one
* Add option to enforce sync with default data
* Default data is added. Old confs removed
* Add *.local into gitignore

Scripts cleanup

Moved from pylibmc to python-memcached

Library pylibmc depends on libmemcached and doesn't work on CentOS (version conflict bw lib requirement and memcached).

Change-Id: I0cc61c6d344ba24442ec954635010b518c0efa95

.gitignore

@@ -1,5 +1,6 @@
 *~
 *.pyc
+*.local
 AUTHORS
 ChangeLog
 MANIFEST

bin/dashboard

@@ -5,7 +5,7 @@ import os
 import sys
 sys.path.insert(0, os.getcwd())
 
-from dashboard.dashboard import app
+from dashboard.web import app
 
 
 app.run()

bin/full-update

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-if [[ -z $STACKALYTICS_HOME ]]; then
-echo "Variable STACKALYTICS_HOME must be specified"
-exit
-fi
-
-echo "Analytics home is $STACKALYTICS_HOME"
-
-DASHBOARD_CONF='$STACKALYTICS_HOME/conf/dashboard.conf'
-
-TOP_DIR=$(cd $(dirname "$0") && pwd)
-
-DB_FILE=`mktemp -u --tmpdir=$STACKALYTICS_HOME/data stackalytics-XXXXXXXXXXXX.sqlite`
-TEMP_CONF=`mktemp -u`
-
-cd $TOP_DIR/../scripts/
-./pull-repos.sh
-
-cd $TOP_DIR/../
-./bin/stackalytics --config-file $STACKALYTICS_HOME/conf/analytics.conf --db-database $DB_FILE --verbose
-
-DATE=`date -u +'%d-%b-%y %H:%M %Z'`
-
-echo DATABASE = \'$DB_FILE\' >> $TEMP_CONF
-echo LAST_UPDATE = \'$DATE\' >> $TEMP_CONF
-
-#rm $DASHBOARD_CONF
-#mv $TEMP_CONF $DASHBOARD_CONF
-
-echo "Data is refreshed, please restart service"

bin/grab-unmapped

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-TOP_DIR=$(cd $(dirname "$0") && pwd)
-
-./tools/with_venv.sh python scripts/launchpad/grab-unmapped-launchpad-ids.py

bin/processor

@@ -5,7 +5,7 @@ import os
 import sys
 sys.path.insert(0, os.getcwd())
 
-from pycvsanaly2.main import main
+from stackalytics.processor.main import main
 
 
 main()

bin/update

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-if [[ -z $STACKALYTICS_HOME ]]; then
-    echo "Variable STACKALYTICS_HOME must be specified"
-    exit
-fi
-
-echo "Analytics home is $STACKALYTICS_HOME"
-
-CONF="$STACKALYTICS_HOME/conf/analytics.conf"
-
-TOP_DIR=$(cd $(dirname "$0") && pwd)
-cd $TOP_DIR/../scripts/
-./pull-repos.sh
-
-echo "Updating data"
-cd $TOP_DIR/../
-./bin/stackalytics --config-file $CONF --db-database $STACKALYTICS_HOME/data/stackalyticss.sqlite --verbose

dashboard/memory_storage.py

@@ -0,0 +1,100 @@
+from stackalytics.processor import user_utils
+
+MEMORY_STORAGE_CACHED = 0
+
+
+class MemoryStorage(object):
+    def __init__(self, records):
+        pass
+
+
+class CachedMemoryStorage(MemoryStorage):
+    def __init__(self, records):
+        super(CachedMemoryStorage, self).__init__(records)
+
+        self.records = {}
+        self.company_index = {}
+        self.date_index = {}
+        self.module_index = {}
+        self.launchpad_id_index = {}
+        self.release_index = {}
+        self.dates = []
+        for record in records:
+            self.records[record['record_id']] = record
+            self.index(record)
+        self.dates = sorted(self.date_index)
+        self.company_name_mapping = dict((c.lower(), c)
+                                         for c in self.company_index.keys())
+
+    def index(self, record):
+
+        self._add_to_index(self.company_index, record, 'company_name')
+        self._add_to_index(self.module_index, record, 'module')
+        self._add_to_index(self.launchpad_id_index, record, 'launchpad_id')
+        self._add_to_index(self.release_index, record, 'release')
+        self._add_to_index(self.date_index, record, 'date')
+
+        record['week'] = user_utils.timestamp_to_week(record['date'])
+        record['loc'] = record['lines_added'] + record['lines_deleted']
+
+    def _add_to_index(self, record_index, record, key):
+        record_key = record[key]
+        if record_key in record_index:
+            record_index[record_key].add(record['record_id'])
+        else:
+            record_index[record_key] = set([record['record_id']])
+
+    def _get_record_ids_from_index(self, items, index):
+        record_ids = set()
+        for item in items:
+            if item not in index:
+                raise Exception('Parameter %s not valid' % item)
+            record_ids |= index[item]
+        return record_ids
+
+    def get_record_ids_by_modules(self, modules):
+        return self._get_record_ids_from_index(modules, self.module_index)
+
+    def get_record_ids_by_companies(self, companies):
+        return self._get_record_ids_from_index(
+            map(self._get_company_name, companies),
+            self.company_index)
+
+    def get_record_ids_by_launchpad_ids(self, launchpad_ids):
+        return self._get_record_ids_from_index(launchpad_ids,
+                                               self.launchpad_id_index)
+
+    def get_record_ids_by_releases(self, releases):
+        return self._get_record_ids_from_index(releases, self.release_index)
+
+    def get_record_ids(self):
+        return set(self.records.keys())
+
+    def get_records(self, record_ids):
+        for i in record_ids:
+            yield self.records[i]
+
+    def _get_company_name(self, company_name):
+        normalized = company_name.lower()
+        if normalized not in self.company_name_mapping:
+            raise Exception('Unknown company name %s' % company_name)
+        return self.company_name_mapping[normalized]
+
+    def get_companies(self):
+        return self.company_index.keys()
+
+    def get_modules(self):
+        return self.module_index.keys()
+
+    def get_launchpad_ids(self):
+        return self.launchpad_id_index.keys()
+
+
+class MemoryStorageFactory(object):
+
+    @staticmethod
+    def get_storage(memory_storage_type, records):
+        if memory_storage_type == MEMORY_STORAGE_CACHED:
+            return CachedMemoryStorage(records)
+        else:
+            raise Exception('Unknown memory storage type')

dashboard/static/css/style.css

@@ -94,7 +94,7 @@ div.drops label {
     color: #909cb5;
 }
 
-span.drop_period {
+span.drop_release {
     margin-top: 6px;
     display: block;
     height: 30px;

dashboard/templates/base.html

@@ -34,9 +34,9 @@
 	<link rel='archives' title='July 2011' href='http://www.mirantis.com/2011/07/' />
 	<link rel='archives' title='June 2011' href='http://www.mirantis.com/2011/06/' />
 	<link rel='archives' title='May 2011' href='http://www.mirantis.com/2011/05/' />
-	<link href='http://fonts.googleapis.com/css?family=PT+Sans:400,700,400italic&subset=latin,cyrillic' rel='stylesheet' type='text/css'>
+	<link href='http://fonts.googleapis.com/css?family=PT+Sans:400,700,400italic&subset=latin,cyrillic' rel='stylesheet' type='text/css' />
-	<link href='http://fonts.googleapis.com/css?family=PT+Sans+Caption&subset=latin,cyrillic' rel='stylesheet' type='text/css'>
+	<link href='http://fonts.googleapis.com/css?family=PT+Sans+Caption&subset=latin,cyrillic' rel='stylesheet' type='text/css' />
-	<link href='http://fonts.googleapis.com/css?family=PT+Sans+Narrow:400,700&subset=latin,cyrillic' rel='stylesheet' type='text/css'>
+	<link href='http://fonts.googleapis.com/css?family=PT+Sans+Narrow:400,700&subset=latin,cyrillic' rel='stylesheet' type='text/css' />
 
 
 {#    <script type="text/javascript" src="{{ url_for('static', filename='js/jquery-1.9.1.min.js') }}"></script>#}
@@ -49,21 +49,21 @@
     {% block head %}{% endblock %}
 
     <!-- Google Analytics -->
-    <script type="text/javascript">
+{#    <script type="text/javascript">#}
-        var _gaq = _gaq || [];
+{#        var _gaq = _gaq || [];#}
-        _gaq.push(['_setAccount', 'UA-8933515-2']);
+{#        _gaq.push(['_setAccount', 'UA-8933515-2']);#}
-        _gaq.push(['_setDomainName', 'stackalytics.com']);
+{#        _gaq.push(['_setDomainName', 'stackalytics.com']);#}
-        _gaq.push(['_setAllowLinker', true]);
+{#        _gaq.push(['_setAllowLinker', true]);#}
-        _gaq.push(['_trackPageview']);
+{#        _gaq.push(['_trackPageview']);#}
-        (function () {
+{#        (function () {#}
-            var ga = document.createElement('script');
+{#            var ga = document.createElement('script');#}
-            ga.type = 'text/javascript';
+{#            ga.type = 'text/javascript';#}
-            ga.async = true;
+{#            ga.async = true;#}
-            ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+{#            ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';#}
-            var s = document.getElementsByTagName('script')[0];
+{#            var s = document.getElementsByTagName('script')[0];#}
-            s.parentNode.insertBefore(ga, s);
+{#            s.parentNode.insertBefore(ga, s);#}
-        })();
+{#        })();#}
-    </script>
+{#    </script>#}
 
 </head>
 

dashboard/templates/company_details.html

@@ -4,16 +4,19 @@
     {{ company }}
 {% endblock %}
 
+{% block scripts %}
+<script type="text/javascript">
+    chartAndTableRenderer("/data/engineers", "left_list", "left_chart", "/engineers/", {company: "{{ company|encode }}" });
+    chartAndTableRenderer("/data/modules", "right_list", "right_chart", "/modules/", {company: "{{ company|encode }}" });
+    timelineRenderer({company: "{{ company|encode }}" })
+</script>
+{% endblock %}
+
+
 {% block left_frame %}
 
     <h2>Contribution by engineers</h2>
 
-    <script type="text/javascript">
-        loadTable("left_list", "/data/companies/{{ company|encode }}");
-        loadChart("left_chart", "/data/companies/{{ company|encode }}", {limit: 10});
-        loadTimeline({company: '{{ company }}'})
-    </script>
-
     <div id="left_chart" style="width: 100%; height: 350px;"></div>
 
     <table id="left_list" class="display">
@@ -35,11 +38,6 @@
 
     <h2>Contribution by modules</h2>
 
-    <script type="text/javascript">
-        loadTable("right_list", "/data/modules", {company: "{{ company|encode }}" });
-        loadChart("right_chart", "/data/modules", {limit: 10, company: "{{ company|encode }}" });
-    </script>
-
     <div id="right_chart" style="width: 100%; height: 350px;"></div>
 
     <table id="right_list" class="display">
@@ -59,10 +57,10 @@
     {%  if blueprints %}
         <div>Blueprints:
             <ol>
-                {% for one in blueprints %}
+                {% for rec in blueprints %}
                     <li>
-                        <a href="https://blueprints.launchpad.net/{{ one[1] }}/+spec/{{ one[0] }}">{{ one[0] }}</a>
+                        <a href="https://blueprints.launchpad.net/{{ rec['module'] }}/+spec/{{ rec['id'] }}">{{ rec['id'] }}</a>
-                        <small>{{ one[1] }}</small>
+                        <small>{{ rec['module'] }}</small>
                     </li>
                 {% endfor %}
             </ol>
@@ -70,12 +68,18 @@
     {% endif %}
 
     {% if bugs %}
-        <div>Bug fixes: <b>{{ bugs|length }}</b>
+        <div>Bugs:
+            <ol>
+                {% for rec in bugs %}
+                    <li>
+                        <a href="https://bugs.launchpad.net/bugs/{{ rec['id'] }}">{{ rec['id'] }}</a>
+                    </li>
+                {% endfor %}
+            </ol>
         </div>
     {% endif %}
 
-    <div>Total commits: <b>{{ commits|length }}</b>, among them <b>{{ code_commits }}</b> commits in code,
+    <div>Total commits: <b>{{ commits|length }}</b></div>
-        among them <b>{{ test_only_commits }}</b> test only commits</div>
     <div>Total LOC: <b>{{ loc }}</b></div>
 
 {% endblock %}

dashboard/templates/engineer_details.html

@@ -1,25 +1,26 @@
 {% extends "layout.html" %}
 
 {% block title %}
-    {{ details.name }}
+    {{ user.user_name }}
+{% endblock %}
+
+{% block scripts %}
+<script type="text/javascript">
+    chartAndTableRenderer("/data/modules", "right_list", "right_chart", "/modules/", {launchpad_id: "{{ launchpad_id }}" });
+    timelineRenderer({launchpad_id: "{{ launchpad_id }}" })
+</script>
 {% endblock %}
 
 {% block left_frame %}
-
-    <script type="text/javascript">
-        loadTimeline({engineer: '{{engineer}}'})
-    </script>
-
     <div style='float: left;'>
-        <img src="{{ details.email|gravatar(size=64) }}">
+        <img src="{{ user.emails[0]|gravatar(size=64) }}">
     </div>
     <div style='margin-left: 90px;'>
-        <h2 style='margin-bottom: 0.5em;'>{{ details.name }}</h2>
+        <h2 style='margin-bottom: 0.5em;'>{{ user.user_name }}</h2>
-        {% if details.company %}
+        {% if user.companies %}
-        <div>Company: {{ link('/companies/' + details.company, details.company) }}</div>
+        <div>Company: {{ user.companies[-1].company_name|link('/companies/' + user.companies[-1].company_name)|safe }}</div>
         {% endif %}
-        <div>Launchpad: <a href="https://launchpad.net/~{{ details.launchpad_id }}">{{ details.launchpad_id }}</a></div>
+        <div>Launchpad: <a href="https://launchpad.net/~{{ launchpad_id }}">{{ launchpad_id }}</a></div>
-{#        <div>Email: {{ details.email }}</div>#}
     </div>
 
     <h3>Commits history</h3>
@@ -28,15 +29,13 @@
         <div>There are no commits for selected period or project type.</div>
     {% endif %}
 
-    {% for message in commits %}
+    {% for rec in commits %}
         <div>
-            <h4>{{ message.date|datetimeformat }} to <a href="https://launchpad.net/{{ message.module }}">{{ message.module }}</a>
+            <h4>{{ rec.date|datetimeformat }} to <a href="https://launchpad.net/{{ rec.module }}">{{ rec.module }}</a>
-                {% if message.is_code %} <span style="color: royalblue">code</span> {% endif %}
-                {% if message.is_test %} <span style="color: magenta">test</span> {% endif %}
             </h4>
-            <div style='white-space: pre-wrap; padding-left: 2em;'>{{ message.message|safe }}</div>
+            <div style='white-space: pre-wrap; padding-left: 2em;'>{{ rec|commit_message|safe }}</div>
-            <div style="padding-left: 2em;"><span style="color: green">+ {{ message.added_loc }}</span>
+            <div style="padding-left: 2em;"><span style="color: green">+ {{ rec.lines_added }}</span>
-                <span style="color: red">- {{ message.removed_loc }}</span></div>
+                <span style="color: red">- {{ rec.lines_deleted }}</span></div>
         </div>
     {% endfor %}
 
@@ -47,11 +46,6 @@
     {% if commits %}
     <h2>Contribution by modules</h2>
 
-    <script type="text/javascript">
-        loadTable("right_list", "/data/modules", {engineer: "{{ engineer }}" });
-        loadChart("right_chart", "/data/modules", {limit: 10, engineer: "{{ engineer }}" });
-    </script>
-
     <div id="right_chart" style="width: 100%; height: 350px;"></div>
 
     <table id="right_list" class="display">
@@ -71,10 +65,10 @@
     {%  if blueprints %}
         <div>Blueprints:
             <ol>
-                {% for one in blueprints %}
+                {% for rec in blueprints %}
                     <li>
-                        <a href="https://blueprints.launchpad.net/{{ one[1] }}/+spec/{{ one[0] }}">{{ one[0] }}</a>
+                        <a href="https://blueprints.launchpad.net/{{ rec['module'] }}/+spec/{{ rec['id'] }}">{{ rec['id'] }}</a>
-                        <small>{{ one[1] }}</small>
+                        <small>{{ rec['module'] }}</small>
                     </li>
                 {% endfor %}
             </ol>
@@ -82,23 +76,18 @@
     {% endif %}
 
     {% if bugs %}
-        <div>Bug fixes:
+        <div>Bugs:
             <ol>
-                {% for one in bugs %}
+                {% for rec in bugs %}
                     <li>
-                        <a href="https://bugs.launchpad.net/bugs/{{ one[0] }}">{{ one[0] }}</a>
+                        <a href="https://bugs.launchpad.net/bugs/{{ rec['id'] }}">{{ rec['id'] }}</a>
-                        <small>
-                            {% if one[1] %} <span style="color: royalblue">C</span> {% endif %}
-                            {% if one[2] %} <span style="color: magenta">T</span> {% endif %}
-                        </small>
                     </li>
                 {% endfor %}
             </ol>
         </div>
     {% endif %}
 
-    <div>Total commits: <b>{{ commits|length }}</b>, among them <b>{{ code_commits }}</b> commits in code,
+    <div>Total commits: <b>{{ commits|length }}</b></div>
-        among them <b>{{ test_only_commits }}</b> test only commits</div>
     <div>Total LOC: <b>{{ loc }}</b></div>
     {% endif %}
 

dashboard/templates/layout.html

@@ -18,95 +18,12 @@
 <script type="text/javascript" src="{{ url_for('static', filename='js/jqplot.highlighter.min.js') }}"></script>
 
 <script type="text/javascript">
-    // load table data
-    function loadTable(table_id, source, options) {
-        $(document).ready(function () {
-            $("#"+table_id).dataTable({
-                "aLengthMenu": [[25, 50, -1], [25, 50, "All"]],
-                "aaSorting": [[ 2, "desc" ]],
-                "bProcessing": true,
-                "sAjaxSource": make_uri(source, options),
-                "sPaginationType": "full_numbers",
-                "iDisplayLength": 25,
-                "aoColumns": [
-                    { "mData": "index" },
-                    { "mData": "link" },
-                    { "mData": "rank" }
-                ]
-            });
-        });
-    }
 
-    // load chart
+    function showTimeline(data) {
-    function loadChart(chart_id, source, options) {
+        var plot = $.jqplot('timeline', data, {
-        $(document).ready(function () {
+            gridPadding: {
-            // Our ajax data renderer which here retrieves a text file.
+                right: 35
-            // it could contact any source and pull data, however.
-            // The options argument isn't used in this renderer.
-            var ajaxDataRenderer = function (url, plot, options) {
-                var ret = null;
-                $.ajax({
-                    // have to use synchronous here, else the function
-                    // will return before the data is fetched
-                    async: false,
-                    url: url,
-                    dataType: "json",
-                    success: function (data) {
-                        var array = [];
-                        for(i = 0; i < data['aaData'].length; i++) {
-                            array.push([data['aaData'][i].name, data['aaData'][i].rank]);
-                        }
-                        ret = [array]
-                    }
-                });
-                return ret;
-            };
-
-            // passing in the url string as the jqPlot data argument is a handy
-            // shortcut for our renderer.  You could also have used the
-            // "dataRendererOptions" option to pass in the url.
-            var plot = $.jqplot(chart_id, make_uri(source, options), {
-                dataRenderer: ajaxDataRenderer,
-                seriesDefaults: {
-                    // Make this a pie chart.
-                    renderer: jQuery.jqplot.PieRenderer,
-                    rendererOptions: {
-                        // Put data labels on the pie slices.
-                        // By default, labels show the percentage of the slice.
-                        showDataLabels: true
-                    }
             },
-                legend: { show: true, location: 'e' }
-            });
-        });
-    }
-
-    // load timeline
-    function loadTimeline(options) {
-        $(document).ready(function () {
-            var ajaxDataRenderer = function (url, plot, options) {
-                var ret = null;
-                $.ajax({
-                    // have to use synchronous here, else the function
-                    // will return before the data is fetched
-                    async: false,
-                    url: url,
-                    dataType: "json",
-                    success: function (data) {
-                        ret = data;
-                    }
-                });
-                return ret;
-            };
-
-            var jsonurl = make_uri("/data/timeline", options);
-
-            var plot = $.jqplot('timeline', jsonurl, {
-                dataRenderer: ajaxDataRenderer,
-                dataRendererOptions: {
-                    unusedOptionalUrl: jsonurl
-                },
-                gridPadding: {right: 35},
             cursor: {
                 show: false
             },
@@ -156,21 +73,118 @@
                 }
             ]
         });
+    }
+
+    function timelineRenderer(options) {
+        $(document).ready(function () {
+            $.ajax({
+                url: make_uri("/data/timeline", options),
+                dataType: "json",
+                success: function (data) {
+                    showTimeline(data);
+                }
             });
+        });
+    }
+
+    function chartAndTableRenderer(url, table_id, chart_id, link_prefix, options) {
+
+        $(document).ready(function () {
+
+            $.ajax({
+                url: make_uri(url, options),
+                dataType: "json",
+                success: function (data) {
+
+                    var tableData = [];
+                    var chartData = [];
+
+                    var limit = 10;
+                    var aggregate = 0;
+                    var index = 1;
+                    var i;
+
+                    for (i = 0; i < data.length; i++) {
+                        if (i < limit - 1) {
+                            chartData.push([data[i].name, data[i].metric]);
+                        } else {
+                            aggregate += data[i].metric;
+                        }
+
+                        var index_label = index;
+                        if (data[i].name == "*independent") {
+                            index_label = "";
+                        } else {
+                            index++;
+                        }
+                        var link = make_link(link_prefix, data[i].id, data[i].name);
+                        tableData.push({"index": index_label, "link": link, "metric": data[i].metric});
+                    }
+
+                    if (i == limit) {
+                        chartData.push([data[i-1].name, data[i-1].metric]);
+                    } else if (i > limit) {
+                        chartData.push(["others", aggregate]);
+                    }
+
+                    $("#" + table_id).dataTable({
+                        "aLengthMenu": [
+                            [25, 50, -1],
+                            [25, 50, "All"]
+                        ],
+                        "aaSorting": [
+                            [ 2, "desc" ]
+                        ],
+                        "sPaginationType": "full_numbers",
+                        "iDisplayLength": 25,
+                        "aaData": tableData,
+                        "aoColumns": [
+                            { "mData": "index" },
+                            { "mData": "link" },
+                            { "mData": "metric" }
+                        ]
+                    });
+
+                    var plot = $.jqplot(chart_id, [chartData], {
+                        seriesDefaults: {
+                            renderer: jQuery.jqplot.PieRenderer,
+                            rendererOptions: {
+                                showDataLabels: true
+                            }
+                        },
+                        legend: { show: true, location: 'e' }
+                    });
+
+                }
+            });
+        });
+    }
+
+    function getUrlVars() {
+        var vars = {};
+        var parts = window.location.href.replace(/[?&]+([^=&]+)=([^&]*)/gi, function (m, key, value) {
+            vars[key] = value;
+        });
+        return vars;
     }
 
     $(document).ready(function () {
         $('#metric').val('{{ metric }}');
-        $('#period').val('{{ period }}');
+        $('#release').val('{{ release }}');
         $('#project_type').val('{{ project_type }}');
     });
 
+    function make_link(uri_prefix, id, title, options) {
+        var link = make_uri(uri_prefix + encodeURIComponent(id).toLowerCase(), options);
+        return "<a href=\"" + link + "\">" + title + "</a>"
+    }
+
     function make_uri(uri, options) {
         var ops = {};
         if (options != null) {
             $.extend(ops, options);
         }
-        $.extend(ops, make_std_options());
+        $.extend(ops, getUrlVars());
         var str = $.map(ops,function (val, index) {
             return index + "=" + val;
         }).join("&");
@@ -180,9 +194,9 @@
 
     function make_std_options() {
         var options = {};
-        if (getPeriod() != 'havana') {
+{#        if (getRelease() != 'havana') {#}
-            options['period'] = getPeriod();
+            options['release'] = getRelease();
-        }
+{#        }#}
         if (getMetric() != 'loc') {
             options['metric'] = getMetric();
         }
@@ -203,7 +217,7 @@
         reload();
     });
 
-    $(document).on('change', '#period', function (evt) {
+    $(document).on('change', '#release', function (evt) {
         reload();
     });
 
@@ -211,8 +225,8 @@
         reload();
     });
 
-    function getPeriod() {
+    function getRelease() {
-        return $('#period').val()
+        return $('#release').val()
     }
 
     function getMetric() {
@@ -225,6 +239,8 @@
 
 </script>
 
+{% block scripts %}{% endblock %}
+
 {% endblock %}
 
 {% block body %}
@@ -244,10 +260,9 @@
                 <option value="loc">Lines of code</option>
             </select>
             </span>
-            <span class="drop_period" style="float: right;">
+            <span class="drop_release" style="float: right;">
-            <label for="period">Period&nbsp;</label><select id="period" name="period">
+            <label for="release">Release&nbsp;</label><select id="release" name="release">
                 <option value="all">All times</option>
-                <option value="six_months">Last 6 months</option>
                 <option value="havana">Havana</option>
                 <option value="grizzly">Grizzly</option>
                 <option value="folsom">Folsom</option>
@@ -278,7 +293,3 @@
 
 </div>
 {% endblock %}
-
-{% macro link(base, title) -%}
-    <a href="{{ base }}?metric={{ metric }}&period={{ period }}&project_type={{ project_type }}">{{ title }}</a>
-{%- endmacro %}

dashboard/templates/module_details.html

@@ -4,16 +4,17 @@
     {{ module }}
 {% endblock %}
 
+{% block scripts %}
+<script type="text/javascript">
+    chartAndTableRenderer("/data/companies", "left_list", "left_chart", "/companies/", {company: "{{ company|encode }}" });
+    timelineRenderer({company: "{{ company|encode }}" })
+</script>
+{% endblock %}
+
 {% block left_frame %}
 
     <h2>Contribution by companies</h2>
 
-    <script type="text/javascript">
-        loadTable("left_list", "/data/companies", { module: "{{ module }}" });
-        loadChart("left_chart", "/data/companies", { module: "{{ module }}", limit: 10 });
-        loadTimeline({module: '{{ module }}'})
-    </script>
-
     <div id="left_chart" style="width: 100%; height: 350px;"></div>
 
     <table id="left_list" class="display">
@@ -38,30 +39,25 @@
     {% for rec in commits %}
         <div style="padding-bottom: 1em;">
             <div style='float: left; '>
-                <img src="{{ rec.email|gravatar(size=32) }}">
+                <img src="{{ rec.author_email|gravatar(size=32) }}">
             </div>
             <div style="margin-left: 4em;">
                 <div>
-                {{ link('/engineers/' + rec.launchpad_id, rec.name) }}
+                {% if rec.launchpad_id %}
-{#                <a href="/engineers/{{ rec.launchpad_id }}?metric={{ metric }}&period={{ period }}&project_type={{ project_type }}">{{ rec.name }}</a>#}
+                    {{ rec.author|link('/engineers/' + rec.launchpad_id)|safe }}
+                {% else %}
+                    {{ rec.author }}
+                {% endif %}
                 {% if rec.company %}
                     (
-                    {{ link('/companies/' + rec.company, rec.company) }}
+                    {{ rec.company|link('/companies/' + rec.company)|safe }}
-{#                    <a href="/companies/{{ rec.company }}?metric={{ metric }}&period={{ period }}&project_type={{ project_type }}">{{ rec.company }}</a>#}
                     )
                 {% endif %}
                 <em>{{ rec.date|datetimeformat }}</em>
                 </div>
 
-                {% if rec.ref %}
+                <div><b>{{ rec.subject }}</b></div>
-                    <div>{{ rec.ref|safe }}</div>
+                <div style="white-space: pre-wrap;">{{ rec|commit_message|safe }}</div>
-                {% endif %}
-                <div>{{ rec.text }}</div>
-                {% if rec.change_id %}
-                    <div>Change-Id: <a
-                            href="https://review.openstack.org/#q,{{ rec.change_id }},n,z">{{ rec.change_id }}</a>
-                    </div>
-                {% endif %}
             </div>
         </div>
     {% endfor %}

dashboard/templates/overview.html

@@ -4,16 +4,18 @@
     Overview
 {% endblock %}
 
+{% block scripts %}
+<script type="text/javascript">
+    chartAndTableRenderer("/data/companies", "left_list", "left_chart", "/companies/");
+    chartAndTableRenderer("/data/modules", "right_list", "right_chart", "/modules/");
+    timelineRenderer()
+</script>
+{% endblock %}
+
 {% block left_frame %}
 
     <h2>Contribution by companies</h2>
 
-    <script type="text/javascript">
-        loadTable("left_list", "/data/companies");
-        loadChart("left_chart", "/data/companies", {limit: 10});
-        loadTimeline('')
-    </script>
-
     <div id="left_chart" style="width: 100%; height: 350px;"></div>
 
     <table id="left_list" class="display">
@@ -35,11 +37,6 @@
 
     <h2>Contribution by modules</h2>
 
-    <script type="text/javascript">
-        loadTable("right_list", "/data/modules");
-        loadChart("right_chart", "/data/modules", {limit: 10});
-    </script>
-
     <div id="right_chart" style="width: 100%; height: 350px;"></div>
 
     <table id="right_list" class="display">

dashboard/web.py

@@ -0,0 +1,452 @@
+import cgi
+import datetime
+import functools
+import json
+import os
+import re
+import urllib
+
+import flask
+from flask.ext import gravatar as gravatar_ext
+import time
+
+from dashboard import memory_storage
+from stackalytics.processor.persistent_storage import PersistentStorageFactory
+from stackalytics.processor.runtime_storage import RuntimeStorageFactory
+from stackalytics.processor import user_utils
+
+DEBUG = True
+RUNTIME_STORAGE_URI = 'memcached://127.0.0.1:11211'
+PERSISTENT_STORAGE_URI = 'mongodb://localhost'
+
+# create our little application :)
+app = flask.Flask(__name__)
+app.config.from_object(__name__)
+app.config.from_envvar('DASHBOARD_CONF', silent=True)
+
+
+def get_vault():
+    vault = getattr(app, 'stackalytics_vault', None)
+    if not vault:
+        vault = {}
+        vault['runtime_storage'] = RuntimeStorageFactory.get_storage(
+            RUNTIME_STORAGE_URI)
+        vault['persistent_storage'] = PersistentStorageFactory.get_storage(
+            PERSISTENT_STORAGE_URI)
+        vault['memory_storage'] = (
+            memory_storage.MemoryStorageFactory.get_storage(
+                memory_storage.MEMORY_STORAGE_CACHED,
+                vault['runtime_storage'].get_update(os.getpid())))
+
+        releases = vault['persistent_storage'].get_releases()
+        vault['releases'] = dict((r['release_name'].lower(), r)
+                                 for r in releases)
+        app.stackalytics_vault = vault
+    return vault
+
+
+def get_memory_storage():
+    return get_vault()['memory_storage']
+
+
+def record_filter(parameter_getter=lambda x: flask.request.args.get(x)):
+    def decorator(f):
+        @functools.wraps(f)
+        def decorated_function(*args, **kwargs):
+
+            vault = get_vault()
+            memory_storage = vault['memory_storage']
+            record_ids = memory_storage.get_record_ids()
+
+            param = parameter_getter('modules')
+            if param:
+                record_ids &= memory_storage.get_record_ids_by_modules(
+                    param.split(','))
+
+            if 'launchpad_id' in kwargs:
+                param = kwargs['launchpad_id']
+            else:
+                param = (parameter_getter('launchpad_id') or
+                         parameter_getter('launchpad_ids'))
+            if param:
+                record_ids &= memory_storage.get_record_ids_by_launchpad_ids(
+                    param.split(','))
+
+            if 'company' in kwargs:
+                param = kwargs['company']
+            else:
+                param = (parameter_getter('company') or
+                         parameter_getter('companies'))
+            if param:
+                record_ids &= memory_storage.get_record_ids_by_companies(
+                    param.split(','))
+
+            param = parameter_getter('release') or parameter_getter('releases')
+            if param:
+                if param != 'all':
+                    record_ids &= memory_storage.get_record_ids_by_releases(
+                        c.lower() for c in param.split(','))
+
+            kwargs['records'] = memory_storage.get_records(record_ids)
+            return f(*args, **kwargs)
+
+        return decorated_function
+
+    return decorator
+
+
+def aggregate_filter():
+    def decorator(f):
+        @functools.wraps(f)
+        def decorated_function(*args, **kwargs):
+
+            metric_filter = lambda r: r['loc']
+            metric_param = flask.request.args.get('metric')
+            if metric_param:
+                metric = metric_param.lower()
+                if metric == 'commits':
+                    metric_filter = lambda r: 1
+                elif metric != 'loc':
+                    raise Exception('Invalid metric %s' % metric)
+
+            kwargs['metric_filter'] = metric_filter
+            return f(*args, **kwargs)
+
+        return decorated_function
+
+    return decorator
+
+
+def exception_handler():
+    def decorator(f):
+        @functools.wraps(f)
+        def decorated_function(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except Exception as e:
+                print e
+                flask.abort(404)
+
+        return decorated_function
+
+    return decorator
+
+
+DEFAULT_METRIC = 'loc'
+DEFAULT_RELEASE = 'havana'
+DEFAULT_PROJECT_TYPE = 'incubation'
+
+INDEPENDENT = '*independent'
+
+METRIC_LABELS = {
+    'loc': 'Lines of code',
+    'commits': 'Commits',
+}
+
+PROJECT_TYPES = {
+    'core': ['core'],
+    'incubation': ['core', 'incubation'],
+    'all': ['core', 'incubation', 'dev'],
+}
+
+ISSUE_TYPES = ['bug', 'blueprint']
+
+DEFAULT_RECORDS_LIMIT = 10
+
+
+def templated(template=None):
+    def decorator(f):
+        @functools.wraps(f)
+        def decorated_function(*args, **kwargs):
+
+            vault = get_vault()
+            template_name = template
+            if template_name is None:
+                template_name = (flask.request.endpoint.replace('.', '/') +
+                                 '.html')
+            ctx = f(*args, **kwargs)
+            if ctx is None:
+                ctx = {}
+
+            # put parameters into template
+            metric = flask.request.args.get('metric')
+            if metric not in METRIC_LABELS:
+                metric = None
+            ctx['metric'] = metric or DEFAULT_METRIC
+            ctx['metric_label'] = METRIC_LABELS[ctx['metric']]
+
+            release = flask.request.args.get('release')
+            releases = vault['releases']
+            if release:
+                release = release.lower()
+                if release not in releases:
+                    release = None
+                else:
+                    release = releases[release]['release_name']
+            ctx['release'] = (release or DEFAULT_RELEASE).lower()
+
+            return flask.render_template(template_name, **ctx)
+
+        return decorated_function
+
+    return decorator
+
+
+@app.route('/')
+@templated()
+def overview():
+    pass
+
+
+def contribution_details(records, limit=DEFAULT_RECORDS_LIMIT):
+    blueprints_map = {}
+    bugs_map = {}
+    commits = []
+    loc = 0
+
+    for record in records:
+        loc += record['loc']
+        commits.append(record)
+        blueprint = record['blueprint_id']
+        if blueprint:
+            if blueprint in blueprints_map:
+                blueprints_map[blueprint].append(record)
+            else:
+                blueprints_map[blueprint] = [record]
+
+        bug = record['bug_id']
+        if bug:
+            if bug in bugs_map:
+                bugs_map[bug].append(record)
+            else:
+                bugs_map[bug] = [record]
+
+    blueprints = sorted([{'id': key,
+                          'module': value[0]['module'],
+                          'records': value}
+                         for key, value in blueprints_map.iteritems()],
+                        key=lambda x: x['id'])
+    bugs = sorted([{'id': key, 'records': value}
+                   for key, value in bugs_map.iteritems()],
+                  key=lambda x: x['id'])
+    commits.sort(key=lambda x: x['date'], reverse=True)
+
+    result = {
+        'blueprints': blueprints,
+        'bugs': bugs,
+        'commits': commits[0:limit],
+        'loc': loc,
+    }
+    return result
+
+
+@app.route('/companies/<company>')
+@exception_handler()
+@templated()
+@record_filter()
+def company_details(company, records):
+    details = contribution_details(records)
+    details['company'] = company
+    return details
+
+
+@app.route('/modules/<module>')
+@exception_handler()
+@templated()
+@record_filter()
+def module_details(module, records):
+    details = contribution_details(records)
+    details['module'] = module
+    return details
+
+
+@app.route('/engineers/<launchpad_id>')
+@exception_handler()
+@templated()
+@record_filter()
+def engineer_details(launchpad_id, records):
+    persistent_storage = get_vault()['persistent_storage']
+    user = list(persistent_storage.get_users(launchpad_id=launchpad_id))[0]
+
+    details = contribution_details(records)
+    details['launchpad_id'] = launchpad_id
+    details['user'] = user
+    return details
+
+
+@app.errorhandler(404)
+def page_not_found(e):
+    return flask.render_template('404.html'), 404
+
+
+def _get_aggregated_stats(records, metric_filter, keys, param_id,
+                          param_title=None):
+    param_title = param_title or param_id
+    result = dict((c, 0) for c in keys)
+    titles = {}
+    for record in records:
+        result[record[param_id]] += metric_filter(record)
+        titles[record[param_id]] = record[param_title]
+
+    response = [{'id': r, 'metric': result[r], 'name': titles[r]}
+                for r in result if result[r]]
+    response.sort(key=lambda x: x['metric'], reverse=True)
+    return response
+
+
+@app.route('/data/companies')
+@exception_handler()
+@record_filter()
+@aggregate_filter()
+def get_companies(records, metric_filter):
+    response = _get_aggregated_stats(records, metric_filter,
+                                     get_memory_storage().get_companies(),
+                                     'company_name')
+    return json.dumps(response)
+
+
+@app.route('/data/modules')
+@exception_handler()
+@record_filter()
+@aggregate_filter()
+def get_modules(records, metric_filter):
+    response = _get_aggregated_stats(records, metric_filter,
+                                     get_memory_storage().get_modules(),
+                                     'module')
+    return json.dumps(response)
+
+
+@app.route('/data/engineers')
+@exception_handler()
+@record_filter()
+@aggregate_filter()
+def get_engineers(records, metric_filter):
+    response = _get_aggregated_stats(records, metric_filter,
+                                     get_memory_storage().get_launchpad_ids(),
+                                     'launchpad_id', 'author')
+    return json.dumps(response)
+
+
+@app.route('/data/timeline')
+@exception_handler()
+@record_filter(parameter_getter=lambda x: flask.request.args.get(x)
+               if (x != "release") and (x != "releases") else None)
+def timeline(records):
+
+    # find start and end dates
+    release_name = flask.request.args.get('release')
+
+    if not release_name:
+        release_name = DEFAULT_RELEASE
+    else:
+        release_name = release_name.lower()
+
+    releases = get_vault()['releases']
+    if release_name not in releases:
+        flask.abort(404)
+    release = releases[release_name]
+
+    start_date = release_start_date = user_utils.timestamp_to_week(
+        user_utils.date_to_timestamp(release['start_date']))
+    end_date = release_end_date = user_utils.timestamp_to_week(
+        user_utils.date_to_timestamp(release['end_date']))
+    now = user_utils.timestamp_to_week(int(time.time()))
+
+    # expand start-end to year if needed
+    if release_end_date - release_start_date < 52:
+        expansion = (52 - (release_end_date - release_start_date)) // 2
+        if release_end_date + expansion < now:
+            end_date += expansion
+        else:
+            end_date = now
+        start_date = end_date - 52
+
+    # empty stats for all weeks in range
+    weeks = range(start_date, end_date)
+    week_stat_loc = dict((c, 0) for c in weeks)
+    week_stat_commits = dict((c, 0) for c in weeks)
+
+    # fill stats with the data
+    for record in records:
+        week = record['week']
+        if week in weeks:
+            week_stat_loc[week] += record['loc']
+            week_stat_commits[week] += 1
+
+    # form arrays in format acceptable to timeline plugin
+    array_loc = []
+    array_commits = []
+    array_commits_hl = []
+
+    for week in weeks:
+        week_str = user_utils.week_to_date(week)
+        array_loc.append([week_str, week_stat_loc[week]])
+        if release_start_date <= week <= release_end_date:
+            array_commits_hl.append([week_str, week_stat_commits[week]])
+        array_commits.append([week_str, week_stat_commits[week]])
+
+    return json.dumps([array_commits, array_commits_hl, array_loc])
+
+
+# Jinja Filters
+
+@app.template_filter('datetimeformat')
+def format_datetime(timestamp):
+    return datetime.datetime.utcfromtimestamp(
+        timestamp).strftime('%d %b %Y @ %H:%M')
+
+
+@app.template_filter('launchpadmodule')
+def format_launchpad_module_link(module):
+    return '<a href="https://launchpad.net/%s">%s</a>' % (module, module)
+
+
+@app.template_filter('encode')
+def safe_encode(s):
+    return urllib.quote_plus(s)
+
+
+@app.template_filter('link')
+def make_link(title, uri=None):
+    return '<a href="%(uri)s">%(title)s</a>' % {'uri': uri, 'title': title}
+
+
+def clear_text(s):
+    return cgi.escape(re.sub(r'\n{2,}', '\n', s, flags=re.MULTILINE))
+
+
+def link_blueprint(s, module):
+    return re.sub(r'(blueprint\s+)([\w-]+)',
+                  r'\1<a href="https://blueprints.launchpad.net/' +
+                  module + r'/+spec/\2">\2</a>',
+                  s, flags=re.IGNORECASE)
+
+
+def link_bug(s):
+    return re.sub(r'(bug\s+)#?([\d]{5,7})',
+                  r'\1<a href="https://bugs.launchpad.net/bugs/\2">\2</a>',
+                  s, flags=re.IGNORECASE)
+
+
+def link_change_id(s):
+    return re.sub(r'\s+(I[0-9a-f]{40})',
+                  r' <a href="https://review.openstack.org/#q,\1,n,z">\1</a>',
+                  s)
+
+
+@app.template_filter('commit_message')
+def make_commit_message(record):
+
+    return link_change_id(link_bug(link_blueprint(clear_text(
+        record['message']), record['module'])))
+
+
+gravatar = gravatar_ext.Gravatar(app,
+                                 size=100,
+                                 rating='g',
+                                 default='wavatar',
+                                 force_default=False,
+                                 force_lower=False)
+
+if __name__ == '__main__':
+    app.run('0.0.0.0')

etc/domain-company

@@ -1,94 +0,0 @@
-# domain  employer  [< yyyy-mm-dd]
-3ds.com Dassault Systèmes
-99cloud.net 99cloud
-alyseo.com Alyseo
-ansolabs.com Rackspace < 2012-07-20
-ansolabs.com Nebula
-atomia.com Atomia
-att.com AT&T
-attinteractive.com AT&T
-bigswitch.com Big Switch Networks
-b1-systems.de BL Systems
-brocade.com Brocade
-bull.net Bull
-canonical.com Canonical
-cern.ch CERN
-cisco.com Cisco Systems
-citrix.com Citrix
-cloud.com Citrix Systems
-cloudbau.de Cloudbau
-cloudscaling.com Cloudscaling
-dell.com Dell
-Dell.com Dell
-denali-systems.com Denali Systems
-dreamhost.com DreamHost
-emc.com EMC
-enovance.com eNovance
-fathomdb.com FathomDB
-gluster.com Red Hat
-griddynamics.com Grid Dynamics
-guardian.co.uk The Guardian
-hds.com Hitachi
-hp.com HP
-huawei.com Huawei
-ibm.com IBM
-inktank.com Inktank
-intel.com Intel
-internap.com Internap
-isi.edu University of Southern Carolina
-ispras.ru ISP RAS
-kt.com KT Corporation
-kth.se Kungliga Tekniska högskolan
-linux.vnet.ibm.com IBM
-locaweb.com.br Locaweb
-lahondaresearch.org La Honda Research
-managedit.ie Managed IT
-mellanox.com Mellanox
-memset.com Memset
-metacloud.com Metacloud
-midokura.com Midokura
-midokura.jp Midokura
-mirantis.com Mirantis
-mirantis.ru Mirantis
-nasa.gov NASA
-nebula.com Nebula
-nexenta.com Nexenta
-nec.co.jp NEC
-cq.jp.nec.com NEC
-da.jp.nec.com NEC
-mxw.nes.nec.co.jp NEC
-mxd.nes.nec.co.jp NEC
-netapp.com NetApp
-nicira.com Nicira
-nimbisservices.com Nimbis Services
-ntt.co.jp NTT
-ntt.com NTT
-nttdata.com NTT
-nttdata.co.jp NTT
-nttmcl.com NTT
-vertex.co.in NTT
-pednape StackOps
-pistoncloud.com Piston Cloud
-rackspace.co.uk Rackspace
-rackspace.com Rackspace
-radware.com Radware
-redhat.com Red Hat
-scality.com Scality
-sdsc.edu San Diego Supercomputer Center
-sina.com SINA
-software.dell.com Dell
-solidfire.com SolidFire
-suse.de SUSE
-suse.com SUSE
-suse.cz SUSE
-swiftstack.com SwiftStack
-thoughtworks.com ThoughtWorks
-umd.edu University of Maryland
-unimelb.edu.au University of Melbourne
-valinux.co.jp VA Linux
-vexxhost.com VexxHost
-virtualtech.jp Virtualtech
-vmware.com VMware
-wikimedia.org Wikimedia Foundation
-yahoo-inc.com Yahoo!
-zadarastorage.com Zadara Storage

etc/email-aliases

@@ -1,257 +0,0 @@
-armamig@gmail.com Armando.Migliaccio@eu.citrix.com
-yogesh.srikrishnan@rackspace.com yogesh.srikrishnan@rackspace.com
-emagana@gmail.com eperdomo@cisco.com eperdomo@dhcp-171-71-119-164.cisco.com
-jeblair@hp.com jeblair@openstack.org
-rohitgarwalla@gmail.com roagarwa@cisco.com
-shweta.ap05@gmail.com shpadubi@cisco.com
-e0ne@e0ne.info ikolodyazhny@mirantis.com
-santhosh.m@thoughtworks.com santhom@thoughtworks.com
-john.garbutt@citrix.com john.garbutt@rackspace.com
-thingee@gmail.com mike.perez@dreamhost.com
-duncan.thomas@gmail.com duncan.thomas@hp.com
-adamg@canonical.com adam.gandelman@canonical.com
-admin@jakedahn.com jake@ansolabs.com
-amesserl@rackspace.com ant@openstack.org
-amigliaccio@internap.com Armando.Migliaccio@eu.citrix.com
-andrew@cloudscaling.com acs@parvuscaptus.com
-anne.gentle@rackspace.com anne@openstack.org
-anne@openstack.org anne@openstack.org
-armando.migliaccio@citrix.com Armando.Migliaccio@eu.citrix.com
-asomya@cisco.com asomya@cisco.com
-bcwaldon@gmail.com brian.waldon@rackspace.com
-bfschott@gmail.com bschott@isi.edu
-Bogott abogott@wikimedia.org
-brian.lamar@gmail.com brian.lamar@rackspace.com
-brian.lamar@rackspace.com brian.lamar@rackspace.com
-cbehrens@codestud.com cbehrens+github@codestud.com
-cbehrens+github@codestud.com cbehrens+github@codestud.com
-chiradeep@chiradeep-lt2 chiradeep@cloud.com
-chmouel@chmouel.com chmouel.boudjnah@rackspace.co.uk
-chmouel@enovance.com chmouel@chmouel.com
-chris.behrens@rackspace.com cbehrens@codestud.com
-chris@slicehost.com chris@pistoncloud.com
-chuck.short@canonical.com zulcss@ubuntu.com
-clayg@clayg-desktop clay.gerrard@gmail.com
-clay.gerrard@gmail.com clay.gerrard@gmail.com
-clay.gerrard@rackspace.com clay.gerrard@gmail.com
-code@term.ie code@term.ie
-corvus@gnu.org jeblair@hp.com
-corvus@inaugust.com jeblair@hp.com
-corywright@gmail.com cory.wright@rackspace.com
-cory.wright@rackspace.com corywright@gmail.com
-dan@nicira.com dan@nicira.com
-dan.prince@rackspace.com dprince@redhat.com
-danwent@dan-xs3-cs dan@nicira.com
-danwent@gmail.com dan@nicira.com
-Dave.Walker@Canonical.com dave.walker@canonical.com
-DaveWalker@ubuntu.com dave.walker@canonical.com
-DaveWalker@ubuntu.com Dave.Walker@canonical.com
-david.goetz@gmail.com david.goetz@rackspace.com
-david.hadas@gmail.com davidh@il.ibm.com
-devcamcar@illian.local devin.carlen@gmail.com
-devnull@brim.net gholt@rackspace.com
-Dietz matt.dietz@rackspace.com
-dolph.mathews@gmail.com dolph.mathews@rackspace.com
-doug.hellmann@gmail.com doug.hellmann@dreamhost.com
-dougw@sdsc.edu dweimer@gmail.com
-dpgoetz@gmail.com david.goetz@rackspace.com
-dt-github@xr7.org dtroyer@gmail.com
-Édouard edouard.thuleau@orange.com
-emellor@silver ewan.mellor@citrix.com
-enugaev@griddynamics.com reldan@oscloud.ru
-florian.hines@gmail.com syn@ronin.io
-gaurav@gluster.com gaurav@gluster.com
-ghe.rivero@gmail.com ghe@debian.org
-ghe.rivero@stackops.com ghe@debian.org
-gholt@brim.net gholt@rackspace.com
-gihub@highbridgellc.com github@highbridgellc.com
-github@anarkystic.com code@term.ie
-github@anarkystic.com github@anarkystic.com
-glange@rackspace.com greglange@gmail.com
-greglange+launchpad@gmail.com greglange@gmail.com
-heut2008@gmail.com yaguang.tang@canonical.com
-higginsd@gmail.com derekh@redhat.com
-ialekseev@griddynamics.com ilyaalekseyev@acm.org
-ilya@oscloud.ru ilyaalekseyev@acm.org
-itoumsn@shayol itoumsn@nttdata.co.jp
-jake@ansolabs.com jake@ansolabs.com
-jake@markupisart.com jake@ansolabs.com
-james.blair@rackspace.com jeblair@hp.com
-jaypipes@gmail.com jaypipes@gmail.com
-jesse@aire.local anotherjesse@gmail.com
-jesse@dancelamb anotherjesse@gmail.com
-jesse@gigantor.local anotherjesse@gmail.com
-jesse@ubuntu anotherjesse@gmail.com
-jian.wen@ubuntu.com jian.wen@canonical.com
-jkearney@nova.(none) josh@jk0.org
-jkearney@nova.(none) josh.kearney@pistoncloud.com
-jmckenty@joshua-mckentys-macbook-pro.local jmckenty@gmail.com
-jmckenty@yyj-dhcp171.corp.flock.com jmckenty@gmail.com
-joe@cloudscaling.com joe@swiftstack.com
-johannes@compute3.221.st johannes.erdfelt@rackspace.com
-johannes@erdfelt.com johannes.erdfelt@rackspace.com
-john.dickinson@rackspace.com me@not.mn
-john.eo@gmail.com john.eo@rackspace.com
-john.griffith8@gmail.com john.griffith@solidfire.com
-john.griffith@solidfire.com john.griffith@solidfire.com
-josh@jk0.org josh.kearney@pistoncloud.com
-josh.kearney@rackspace.com josh@jk0.org
-josh.kearney@rackspace.com josh.kearney@pistoncloud.com
-joshua.mckenty@nasa.gov jmckenty@gmail.com
-jpipes@serialcoder jaypipes@gmail.com
-jpipes@uberbox.gateway.2wire.net jaypipes@gmail.com
-jsuh@bespin jsuh@isi.edu
-jtran@attinteractive.com jhtran@att.com
-julien.danjou@enovance.com julien@danjou.info
-justin@fathomdb.com justin@fathomdb.com
-justinsb@justinsb-desktop justin@fathomdb.com
-kapil.foss@gmail.com kapil.foss@gmail.com
-ken.pepple@rabbityard.com ken.pepple@gmail.com
-ke.wu@nebula.com ke.wu@ibeca.me
-khaled.hussein@gmail.com khaled.hussein@rackspace.com
-Knouff philip.knouff@mailtrust.com
-Kölker jason@koelker.net
-kshileev@griddynamics.com kshileev@gmail.com
-laner@controller rlane@wikimedia.org
-letterj@racklabs.com letterj@gmail.com
-liem.m.nguyen@gmail.com liem_m_nguyen@hp.com
-liem.m.nguyen@hp.com liem_m_nguyen@hp.com
-Lopez aloga@ifca.unican.es
-lorin@isi.edu lorin@nimbisservices.com
-lrqrun@gmail.com lrqrun@gmail.com
-lzyeval@gmail.com zhongyue.nah@intel.com
-marcelo.martins@rackspace.com btorch@gmail.com
-masumotok@nttdata.co.jp masumotok@nttdata.co.jp
-masumoto masumotok@nttdata.co.jp
-matt.dietz@rackspace.com matt.dietz@rackspace.com
-matthew.dietz@gmail.com matt.dietz@rackspace.com
-matthewdietz@Matthew-Dietzs-MacBook-Pro.local matt.dietz@rackspace.com
-McConnell bmcconne@rackspace.com
-mdietz@openstack matt.dietz@rackspace.com
-mgius7096@gmail.com launchpad@markgius.com
-michael.barton@rackspace.com mike@weirdlooking.com
-michael.still@canonical.com mikal@stillhq.com
-mike-launchpad@weirdlooking.com mike@weirdlooking.com
-Moore joelbm24@gmail.com
-mordred@hudson mordred@inaugust.com
-nati.ueno@gmail.com ueno.nachi@lab.ntt.co.jp
-naveed.massjouni@rackspace.com naveedm9@gmail.com
-nelson@nelson-laptop russ@crynwr.com
-nirmal.ranganathan@rackspace.com rnirmal@gmail.com
-nirmal.ranganathan@rackspace.coom rnirmal@gmail.com
-nova@u4 ueno.nachi@lab.ntt.co.jp
-nsokolov@griddynamics.net nsokolov@griddynamics.com
-openstack@lab.ntt.co.jp ueno.nachi@lab.ntt.co.jp
-paul@openstack.org paul@openstack.org
-paul@substation9.com paul@openstack.org
-paul.voccio@rackspace.com paul@openstack.org
-pvoccio@castor.local paul@openstack.org
-ramana@venus.lekha.org rjuvvadi@hcl.com
-rclark@chat-blanc rick@openstack.org
-renuka.apte@citrix.com renuka.apte@citrix.com
-rick.harris@rackspace.com rconradharris@gmail.com
-rick@quasar.racklabs.com rconradharris@gmail.com
-root@bsirish.(none) sirish.bitra@gmail.com
-root@debian.ohthree.com amesserl@rackspace.com
-root@mirror.nasanebula.net vishvananda@gmail.com
-root@openstack2-api masumotok@nttdata.co.jp
-root@tonbuntu sleepsonthefloor@gmail.com
-root@ubuntu vishvananda@gmail.com
-rrjuvvadi@gmail.com rjuvvadi@hcl.com
-salv.orlando@gmail.com salvatore.orlando@eu.citrix.com
-sandy@sandywalsh.com sandy@darksecretsoftware.com
-sandy@sandywalsh.com sandy.walsh@rackspace.com
-sandy.walsh@rackspace.com sandy@darksecretsoftware.com
-sandy.walsh@rackspace.com sandy.walsh@rackspace.com
-sateesh.chodapuneedi@citrix.com sateesh.chodapuneedi@citrix.com
-SB justin@fathomdb.com
-sirish.bitra@gmail.com sirish.bitra@gmail.com
-sleepsonthefloor@gmail.com sleepsonthefloor@gmail.com
-Smith code@term.ie
-Sokolov nsokolov@griddynamics.com
-Somya asomya@cisco.com
-soren.hansen@rackspace.com soren@linux2go.dk
-soren@linux2go.dk soren.hansen@rackspace.com
-soren@openstack.org soren.hansen@rackspace.com
-sorhanse@cisco.com sorenhansen@rackspace.com
-sorlando@nicira.com salvatore.orlando@eu.citrix.com
-spam@andcheese.org sam@swiftstack.com
-superstack@superstack.org justin@fathomdb.com
-termie@preciousroy.local code@term.ie
-thuleau@gmail.com edouard1.thuleau@orange.com
-thuleau@gmail.com edouard.thuleau@orange.com
-tim.simpson4@gmail.com tim.simpson@rackspace.com
-todd@lapex todd@ansolabs.com
-todd@rubidine.com todd@ansolabs.com
-todd@rubidine.com xtoddx@gmail.com
-tpatil@vertex.co.in tushar.vitthal.patil@gmail.com
-Tran jtran@attinteractive.com
-treyemorris@gmail.com trey.morris@rackspace.com
-ttcl@mac.com troy.toman@rackspace.com
-Ueno ueno.nachi@lab.ntt.co.jp
-vishvananda@yahoo.com vishvananda@gmail.com
-vito.ordaz@gmail.com victor.rodionov@nexenta.com
-wenjianhn@gmail.com jian.wen@canonical.com
-will.wolf@rackspace.com throughnothing@gmail.com
-wwkeyboard@gmail.com aaron.lee@rackspace.com
-xchu@redhat.com xychu2008@gmail.com
-xtoddx@gmail.com todd@ansolabs.com
-xyj.asmy@gmail.com xyj.asmy@gmail.com
-yorik@ytaraday yorik.sar@gmail.com
-YS vivek.ys@gmail.com
-z-github@brim.net gholt@rackspace.com
-ziad.sawalha@rackspace.com github@highbridgellc.com
-z-launchpad@brim.net gholt@rackspace.com
-derek.morton25@gmail.com derek@networkwhisperer.com
-bartosz.gorski@ntti3.com bartosz.gorski@nttmcl.com
-launchpad@chmouel.com chmouel@chmouel.com
-launchpad@chmouel.com chmouel@enovance.com
-launchpad@chmouel.com chmouel@openstack.org
-imsplitbit@gmail.com dsalinas@rackspace.com
-clint@fewbar.com clint.byrum@hp.com
-clint@fewbar.com clint@ubuntu.com
-sbaker@redhat.com steve@stevebaker.org
-asalkeld@redhat.com angus@salkeld.id.au
-evgeniy@afonichev.com eafonichev@mirantis.com
-smoser@ubuntu.com scott.moser@canonical.com
-smoser@ubuntu.com smoser@brickies.net
-smoser@ubuntu.com smoser@canonical.com
-smoser@ubuntu.com ssmoser2@gmail.com
-jason@koelker.net jkoelker@rackspace.com
-john.garbutt@rackspace.com john@johngarbutt.com
-zhongyue.nah@intel.com lzyeval@gmail.com
-jiajun@unitedstack.com iamljj@gmail.com
-christophe.sauthier@ubuntu.com christophe.sauthier@gmail.com
-christophe.sauthier@ubuntu.com christophe@sauthier.com
-christophe.sauthier@objectif-libre.com christophe@sauthier.com
-aababilov@griddynamics.com ilovegnulinux@gmail.com
-yportnova@griddynamics.com yportnov@yahoo-inc.com
-mkislins@yahoo-inc.com mkislinska@griddynamics.com
-ryan.moore@hp.com rmoore08@gmail.com
-starodubcevna@gmail.com nstarodubtsev@mirantis.com
-lakhinder.walia@hds.com lakhindr@hotmail.com
-kanzhe@gmail.com kanzhe.jiang@bigswitch.com
-anita.kuno@enovance.com akuno@lavabit.com
-me@frostman.ru slukjanov@mirantis.com
-alexei.kornienko@gmail.com akornienko@mirantis.com
-nicolas@barcet.com nick.barcet@canonical.com
-nicolas@barcet.com nick@enovance.com
-nicolas@barcet.com nijaba@ubuntu.com
-graham.binns@canonical.com gmb@canonical.com
-graham.binns@canonical.com gmb@grahambinns.com
-graham.binns@canonical.com graham.binns@gmail.com
-graham.binns@canonical.com graham@canonical.com
-graham.binns@canonical.com graham@grahambinns.com
-emilien.macchi@stackops.com emilien.macchi@enovance.com
-emilien.macchi@stackops.com emilien@enovance.com
-swann.croiset@bull.net swann@oopss.org
-soulascedric@gmail.com cedric.soulas@cloudwatt.com
-simon.pasquier@bull.net pasquier.simon+launchpad@gmail.com
-simon.pasquier@bull.net pasquier.simon@gmail.com
-bogorodskiy@gmail.com novel@FreeBSD.org
-bogorodskiy@gmail.com rbogorodskiy@mirantis.com
-svilgelm@mirantis.com sergey.vilgelm@gmail.com
-robert.myers@rackspace.com robert_myers@earthlink.net
-raymond_pekowski@dell.com pekowski@gmail.com
-agorodnev@mirantis.com a.gorodnev@gmail.com
-rprikhodchenko@mirantis.com me@romcheg.me

etc/email-company

@@ -1,17 +0,0 @@
-# user@domain  employer
-anotherjesse@gmail.com Nebula
-bcwaldon@gmail.com Nebula
-code@term.ie Nebula
-dprince@redhat.com Red Hat
-github@anarkystic.com Nebula
-jake@ansolabs.com Nebula
-jaypipes@gmail.com AT&T
-jeblair@hp.com HP
-lzyeval@gmail.com Intel
-me@not.mn SwiftStack
-mordred@inaugust.com HP
-sleepsonthefloor@gmail.com Nebula
-soren@linux2go.dk Cisco
-vishvananda@gmail.com Nebula
-dtroyer@gmail.com Nebula
-

etc/launchpad-company

@@ -1,36 +0,0 @@
-tatyana-leontovich Grid Dynamics
-vkhomenko Grid Dynamics
-cthiel-suse DE Telekom
-yorik-sar Mirantis
-gelbuhos Mirantis
-aababilov Grid Dynamics
-alexpilotti Cloudbase Solutions
-devananda HP
-heckj Nebula
-matt-sherborne Rackspace
-michael-ogorman Cisco Systems
-boris-42 Mirantis
-boris-42 *independent < 2013-04-10
-victor-r-howard Comcast
-amitry Comcast
-scollins Comcast
-w-emailme Comcast
-jasondunsmore Rackspace
-kannan Rightscale
-bob-melander Cisco Systems
-gabriel-hurley Nebula
-mathrock National Security Agency
-yosshy NEC
-johngarbutt Rackspace
-johngarbutt Citrix < 2013-02-01
-jean-baptiste-ransy Alyseo
-darren-birkett Rackspace
-lucasagomes Red Hat
-nobodycam HP
-cboylan HP
-dmllr SUSE
-therve HP
-hughsaunders Rackspace
-bruno-semperlotti Dassault Systèmes
-james-slagle Red Hat
-openstack *robots

etc/openstack.conf

@@ -1,27 +0,0 @@
-[DEFAULT]
-# Run in debug mode?
-# debug = False
-
-# Database parameters
-# db_driver = sqlite
-# db_user = operator
-# db_password = None
-# db_database = /opt/stack/data/stackalytics.sqlite
-# db_hostname = localhost
-
-# Extensions
-# extensions = CommitsLOC,MessageDetails
-
-# Root for all project sources. The tool will iterate over its contents
-# sources_root = /opt/stack/repos
-
-# email mappings (e.g. collected from .mailmap files)
-# email_aliases = etc/email-aliases
-# mappings from domains to company names
-# domain2company = etc/domain-company
-# mappings from emails to company names
-# email2company = etc/email-company
-# mappings from launchpad ids to emails and user names
-# launchpad2email = etc/launchpad-ids
-# mappings from launchpad id to company name
-# launchpad2company = etc/launchpad-company

etc/stackalytics.conf

@@ -0,0 +1,24 @@
+[DEFAULT]
+# Run in debug mode?
+# debug = False
+
+# Default data
+# default-data = etc/default_data.json
+
+# The folder that holds all project sources to analyze
+# sources_root = ../metric-root-tmp
+
+# Runtime storage URI
+# runtime_storage_uri = memcached://127.0.0.1:11211
+
+# URI of persistent storage
+# persistent_storage_uri = mongodb://localhost
+
+# Update persistent storage with default data
+# read-default-data = False
+
+# Repo poll period in seconds
+# repo_poll_period = 300
+
+# Address of update handler
+# frontend_update_address = http://user:user@localhost/update/%s

etc/test_default_data.json

@@ -0,0 +1,121 @@
+{
+    "users": [
+        {
+            "launchpad_id": "foo",
+            "user_name": "Pupkin",
+            "emails": ["a@a"],
+            "companies": [
+                {
+                    "company_name": "Uno",
+                    "end_date": "2013-Jan-01"
+                },
+                {
+                    "company_name": "Duo",
+                    "end_date": null
+                }
+            ]
+        }
+    ],
+
+    "companies": [
+        {
+            "company_name": "Mirantis",
+            "domains": ["mirantis.com"]
+        },
+        {
+            "company_name": "*independent",
+            "domains": [""]
+        },
+        {
+            "company_name": "Hewlett-Packard",
+            "domains": ["hp.com"]
+        },
+        {
+            "company_name": "Intel",
+            "domains": ["intel.com"]
+        }
+    ],
+
+    "repos": [
+        {
+            "branches": ["master"],
+            "name": "Quantum Client",
+            "type": "core",
+            "uri": "git://github.com/openstack/python-quantumclient.git",
+            "releases": [
+                {
+                    "release_name": "Folsom",
+                    "tag_from": "folsom-1",
+                    "tag_to": "2.1"
+                },
+                {
+                    "release_name": "Grizzly",
+                    "tag_from": "2.1",
+                    "tag_to": "2.2.1"
+                },
+                {
+                    "release_name": "Havana",
+                    "tag_from": "2.2.1",
+                    "tag_to": "HEAD"
+                }
+            ]
+        },
+        {
+            "branches": ["master"],
+            "name": "Keystone",
+            "type": "core",
+            "uri": "git://github.com/openstack/keystone.git",
+            "releases": [
+                {
+                    "release_name": "Essex",
+                    "tag_from": "2011.3",
+                    "tag_to": "2012.1"
+                },
+                {
+                    "release_name": "Folsom",
+                    "tag_from": "2012.1",
+                    "tag_to": "2012.2"
+                },
+                {
+                    "release_name": "Grizzly",
+                    "tag_from": "2012.2",
+                    "tag_to": "2013.1"
+                },
+                {
+                    "release_name": "Havana",
+                    "tag_from": "2013.1",
+                    "tag_to": "HEAD"
+                }
+            ]
+        }
+    ],
+
+    "releases": [
+        {
+            "release_name": "ALL",
+            "start_date": "2010-May-01",
+            "end_date": "now"
+        },
+        {
+            "release_name": "Essex",
+            "start_date": "2011-Oct-01",
+            "end_date": "2012-Apr-01"
+        },
+        {
+            "release_name": "Folsom",
+            "start_date": "2012-Apr-01",
+            "end_date": "2012-Oct-01"
+        },
+        {
+            "release_name": "Grizzly",
+            "start_date": "2012-Oct-01",
+            "end_date": "2013-Apr-01"
+        },
+        {
+            "release_name": "Havana",
+            "start_date": "2013-Apr-01",
+            "end_date": "now"
+        }
+    ]
+
+}

requirements.txt

@@ -1,11 +1,12 @@
 d2to1>=0.2.10,<0.3
-pbr>=0.5.16,<0.6
-#MySQL-python
-#pysqlite
-#git+git://github.com/MetricsGrimoire/RepositoryHandler.git#egg=repositoryhandler-0.5
-#git+git://github.com/SoftwareIntrospectionLab/guilty.git#egg=guilty-2.1
-launchpadlib
 Flask>=0.9
 Flask-Gravatar
-oslo.config
+iso8601
-pylibmc
+launchpadlib
+http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a2.tar.gz#egg=oslo.config-1.2.0a2
+pbr>=0.5.16,<0.6
+psutil
+python-memcached
+pymongo
+sh
+six

stackalytics/__init__.py

@@ -0,0 +1 @@
+__author__ = 'ishakhat'

stackalytics/openstack/__init__.py

@@ -0,0 +1 @@
+__author__ = 'ishakhat'

stackalytics/openstack/common/importutils.py

@@ -0,0 +1,68 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Import related utilities and helper functions.
+"""
+
+import sys
+import traceback
+
+
+def import_class(import_str):
+    """Returns a class from a string including module and class."""
+    mod_str, _sep, class_str = import_str.rpartition('.')
+    try:
+        __import__(mod_str)
+        return getattr(sys.modules[mod_str], class_str)
+    except (ValueError, AttributeError):
+        raise ImportError('Class %s cannot be found (%s)' %
+                          (class_str,
+                           traceback.format_exception(*sys.exc_info())))
+
+
+def import_object(import_str, *args, **kwargs):
+    """Import a class and return an instance of it."""
+    return import_class(import_str)(*args, **kwargs)
+
+
+def import_object_ns(name_space, import_str, *args, **kwargs):
+    """Tries to import object from default namespace.
+
+Imports a class and return an instance of it, first by trying
+to find the class in a default namespace, then failing back to
+a full path if not found in the default namespace.
+"""
+    import_value = "%s.%s" % (name_space, import_str)
+    try:
+        return import_class(import_value)(*args, **kwargs)
+    except ImportError:
+        return import_class(import_str)(*args, **kwargs)
+
+
+def import_module(import_str):
+    """Import a module."""
+    __import__(import_str)
+    return sys.modules[import_str]
+
+
+def try_import(import_str, default=None):
+    """Try to import a module and if it fails return default."""
+    try:
+        return import_module(import_str)
+    except ImportError:
+        return default

stackalytics/openstack/common/jsonutils.py

@@ -0,0 +1,169 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+'''
+JSON related utilities.
+
+This module provides a few things:
+
+    1) A handy function for getting an object down to something that can be
+    JSON serialized.  See to_primitive().
+
+    2) Wrappers around loads() and dumps().  The dumps() wrapper will
+    automatically use to_primitive() for you if needed.
+
+    3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
+    is available.
+'''
+
+
+import datetime
+import functools
+import inspect
+import itertools
+import json
+import types
+import xmlrpclib
+
+import six
+
+from stackalytics.openstack.common import timeutils
+
+
+_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
+                     inspect.isfunction, inspect.isgeneratorfunction,
+                     inspect.isgenerator, inspect.istraceback, inspect.isframe,
+                     inspect.iscode, inspect.isbuiltin, inspect.isroutine,
+                     inspect.isabstract]
+
+_simple_types = (types.NoneType, int, basestring, bool, float, long)
+
+
+def to_primitive(value, convert_instances=False, convert_datetime=True,
+                 level=0, max_depth=3):
+    """Convert a complex object into primitives.
+
+    Handy for JSON serialization. We can optionally handle instances,
+    but since this is a recursive function, we could have cyclical
+    data structures.
+
+    To handle cyclical data structures we could track the actual objects
+    visited in a set, but not all objects are hashable. Instead we just
+    track the depth of the object inspections and don't go too deep.
+
+    Therefore, convert_instances=True is lossy ... be aware.
+
+    """
+    # handle obvious types first - order of basic types determined by running
+    # full tests on nova project, resulting in the following counts:
+    # 572754 <type 'NoneType'>
+    # 460353 <type 'int'>
+    # 379632 <type 'unicode'>
+    # 274610 <type 'str'>
+    # 199918 <type 'dict'>
+    # 114200 <type 'datetime.datetime'>
+    #  51817 <type 'bool'>
+    #  26164 <type 'list'>
+    #   6491 <type 'float'>
+    #    283 <type 'tuple'>
+    #     19 <type 'long'>
+    if isinstance(value, _simple_types):
+        return value
+
+    if isinstance(value, datetime.datetime):
+        if convert_datetime:
+            return timeutils.strtime(value)
+        else:
+            return value
+
+    # value of itertools.count doesn't get caught by nasty_type_tests
+    # and results in infinite loop when list(value) is called.
+    if type(value) == itertools.count:
+        return six.text_type(value)
+
+    # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
+    #              tests that raise an exception in a mocked method that
+    #              has a @wrap_exception with a notifier will fail. If
+    #              we up the dependency to 0.5.4 (when it is released) we
+    #              can remove this workaround.
+    if getattr(value, '__module__', None) == 'mox':
+        return 'mock'
+
+    if level > max_depth:
+        return '?'
+
+    # The try block may not be necessary after the class check above,
+    # but just in case ...
+    try:
+        recursive = functools.partial(to_primitive,
+                                      convert_instances=convert_instances,
+                                      convert_datetime=convert_datetime,
+                                      level=level,
+                                      max_depth=max_depth)
+        if isinstance(value, dict):
+            return dict((k, recursive(v)) for k, v in value.iteritems())
+        elif isinstance(value, (list, tuple)):
+            return [recursive(lv) for lv in value]
+
+        # It's not clear why xmlrpclib created their own DateTime type, but
+        # for our purposes, make it a datetime type which is explicitly
+        # handled
+        if isinstance(value, xmlrpclib.DateTime):
+            value = datetime.datetime(*tuple(value.timetuple())[:6])
+
+        if convert_datetime and isinstance(value, datetime.datetime):
+            return timeutils.strtime(value)
+        elif hasattr(value, 'iteritems'):
+            return recursive(dict(value.iteritems()), level=level + 1)
+        elif hasattr(value, '__iter__'):
+            return recursive(list(value))
+        elif convert_instances and hasattr(value, '__dict__'):
+            # Likely an instance of something. Watch for cycles.
+            # Ignore class member vars.
+            return recursive(value.__dict__, level=level + 1)
+        else:
+            if any(test(value) for test in _nasty_type_tests):
+                return six.text_type(value)
+            return value
+    except TypeError:
+        # Class objects are tricky since they may define something like
+        # __iter__ defined but it isn't callable as list().
+        return six.text_type(value)
+
+
+def dumps(value, default=to_primitive, **kwargs):
+    return json.dumps(value, default=default, **kwargs)
+
+
+def loads(s):
+    return json.loads(s)
+
+
+def load(s):
+    return json.load(s)
+
+
+try:
+    import anyjson
+except ImportError:
+    pass
+else:
+    anyjson._modules.append((__name__, 'dumps', TypeError,
+                                       'loads', ValueError, 'load'))
+    anyjson.force_implementation(__name__)

stackalytics/openstack/common/log.py

@@ -0,0 +1,559 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack Foundation.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Openstack logging handler.
+
+This module adds to logging functionality by adding the option to specify
+a context object when calling the various log methods.  If the context object
+is not specified, default formatting is used. Additionally, an instance uuid
+may be passed as part of the log message, which is intended to make it easier
+for admins to find messages related to a specific instance.
+
+It also allows setting of formatting information through conf.
+
+"""
+
+import ConfigParser
+import cStringIO
+import inspect
+import itertools
+import logging
+import logging.config
+import logging.handlers
+import os
+import sys
+import traceback
+
+from oslo.config import cfg
+
+# from quantum.openstack.common.gettextutils import _
+from stackalytics.openstack.common import importutils
+from stackalytics.openstack.common import jsonutils
+# from quantum.openstack.common import local
+
+
+_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+common_cli_opts = [
+    cfg.BoolOpt('debug',
+                short='d',
+                default=False,
+                help='Print debugging output (set logging level to '
+                     'DEBUG instead of default WARNING level).'),
+    cfg.BoolOpt('verbose',
+                short='v',
+                default=False,
+                help='Print more verbose output (set logging level to '
+                     'INFO instead of default WARNING level).'),
+]
+
+logging_cli_opts = [
+    cfg.StrOpt('log-config',
+               metavar='PATH',
+               help='If this option is specified, the logging configuration '
+                    'file specified is used and overrides any other logging '
+                    'options specified. Please see the Python logging module '
+                    'documentation for details on logging configuration '
+                    'files.'),
+    cfg.StrOpt('log-format',
+               default=None,
+               metavar='FORMAT',
+               help='A logging.Formatter log message format string which may '
+                    'use any of the available logging.LogRecord attributes. '
+                    'This option is deprecated.  Please use '
+                    'logging_context_format_string and '
+                    'logging_default_format_string instead.'),
+    cfg.StrOpt('log-date-format',
+               default=_DEFAULT_LOG_DATE_FORMAT,
+               metavar='DATE_FORMAT',
+               help='Format string for %%(asctime)s in log records. '
+                    'Default: %(default)s'),
+    cfg.StrOpt('log-file',
+               metavar='PATH',
+               deprecated_name='logfile',
+               help='(Optional) Name of log file to output to. '
+                    'If no default is set, logging will go to stdout.'),
+    cfg.StrOpt('log-dir',
+               deprecated_name='logdir',
+               help='(Optional) The base directory used for relative '
+                    '--log-file paths'),
+    cfg.BoolOpt('use-syslog',
+                default=False,
+                help='Use syslog for logging.'),
+    cfg.StrOpt('syslog-log-facility',
+               default='LOG_USER',
+               help='syslog facility to receive log lines')
+]
+
+generic_log_opts = [
+    cfg.BoolOpt('use_stderr',
+                default=True,
+                help='Log output to standard error')
+]
+
+log_opts = [
+    cfg.StrOpt('logging_context_format_string',
+               default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
+                       '%(name)s [%(request_id)s %(user)s %(tenant)s] '
+                       '%(instance)s%(message)s',
+               help='format string to use for log messages with context'),
+    cfg.StrOpt('logging_default_format_string',
+               default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
+                       '%(name)s [-] %(instance)s%(message)s',
+               help='format string to use for log messages without context'),
+    cfg.StrOpt('logging_debug_format_suffix',
+               default='%(funcName)s %(pathname)s:%(lineno)d',
+               help='data to append to log format when level is DEBUG'),
+    cfg.StrOpt('logging_exception_prefix',
+               default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
+               '%(instance)s',
+               help='prefix each line of exception output with this format'),
+    cfg.ListOpt('default_log_levels',
+                default=[
+                    'amqplib=WARN',
+                    'sqlalchemy=WARN',
+                    'boto=WARN',
+                    'suds=INFO',
+                    'keystone=INFO',
+                    'eventlet.wsgi.server=WARN'
+                ],
+                help='list of logger=LEVEL pairs'),
+    cfg.BoolOpt('publish_errors',
+                default=False,
+                help='publish error events'),
+    cfg.BoolOpt('fatal_deprecations',
+                default=False,
+                help='make deprecations fatal'),
+
+    # NOTE(mikal): there are two options here because sometimes we are handed
+    # a full instance (and could include more information), and other times we
+    # are just handed a UUID for the instance.
+    cfg.StrOpt('instance_format',
+               default='[instance: %(uuid)s] ',
+               help='If an instance is passed with the log message, format '
+                    'it like this'),
+    cfg.StrOpt('instance_uuid_format',
+               default='[instance: %(uuid)s] ',
+               help='If an instance UUID is passed with the log message, '
+                    'format it like this'),
+]
+
+CONF = cfg.CONF
+CONF.register_cli_opts(common_cli_opts)
+CONF.register_cli_opts(logging_cli_opts)
+CONF.register_opts(generic_log_opts)
+CONF.register_opts(log_opts)
+
+# our new audit level
+# NOTE(jkoelker) Since we synthesized an audit level, make the logging
+#                module aware of it so it acts like other levels.
+logging.AUDIT = logging.INFO + 1
+logging.addLevelName(logging.AUDIT, 'AUDIT')
+
+
+try:
+    NullHandler = logging.NullHandler
+except AttributeError:  # NOTE(jkoelker) NullHandler added in Python 2.7
+    class NullHandler(logging.Handler):
+        def handle(self, record):
+            pass
+
+        def emit(self, record):
+            pass
+
+        def createLock(self):
+            self.lock = None
+
+
+def _dictify_context(context):
+    if context is None:
+        return None
+    if not isinstance(context, dict) and getattr(context, 'to_dict', None):
+        context = context.to_dict()
+    return context
+
+
+def _get_binary_name():
+    return os.path.basename(inspect.stack()[-1][1])
+
+
+def _get_log_file_path(binary=None):
+    logfile = CONF.log_file
+    logdir = CONF.log_dir
+
+    if logfile and not logdir:
+        return logfile
+
+    if logfile and logdir:
+        return os.path.join(logdir, logfile)
+
+    if logdir:
+        binary = binary or _get_binary_name()
+        return '%s.log' % (os.path.join(logdir, binary),)
+
+
+class BaseLoggerAdapter(logging.LoggerAdapter):
+
+    def audit(self, msg, *args, **kwargs):
+        self.log(logging.AUDIT, msg, *args, **kwargs)
+
+
+class LazyAdapter(BaseLoggerAdapter):
+    def __init__(self, name='unknown', version='unknown'):
+        self._logger = None
+        self.extra = {}
+        self.name = name
+        self.version = version
+
+    @property
+    def logger(self):
+        if not self._logger:
+            self._logger = getLogger(self.name, self.version)
+        return self._logger
+
+
+class ContextAdapter(BaseLoggerAdapter):
+    warn = logging.LoggerAdapter.warning
+
+    def __init__(self, logger, project_name, version_string):
+        self.logger = logger
+        self.project = project_name
+        self.version = version_string
+
+    @property
+    def handlers(self):
+        return self.logger.handlers
+
+    def deprecated(self, msg, *args, **kwargs):
+        stdmsg = _("Deprecated: %s") % msg
+        if CONF.fatal_deprecations:
+            self.critical(stdmsg, *args, **kwargs)
+            raise DeprecatedConfig(msg=stdmsg)
+        else:
+            self.warn(stdmsg, *args, **kwargs)
+
+    def process(self, msg, kwargs):
+        if 'extra' not in kwargs:
+            kwargs['extra'] = {}
+        extra = kwargs['extra']
+
+        context = kwargs.pop('context', None)
+        # if not context:
+        #     context = getattr(local.store, 'context', None)
+        if context:
+            extra.update(_dictify_context(context))
+
+        instance = kwargs.pop('instance', None)
+        instance_extra = ''
+        if instance:
+            instance_extra = CONF.instance_format % instance
+        else:
+            instance_uuid = kwargs.pop('instance_uuid', None)
+            if instance_uuid:
+                instance_extra = (CONF.instance_uuid_format
+                                  % {'uuid': instance_uuid})
+        extra.update({'instance': instance_extra})
+
+        extra.update({"project": self.project})
+        extra.update({"version": self.version})
+        extra['extra'] = extra.copy()
+        return msg, kwargs
+
+
+class JSONFormatter(logging.Formatter):
+    def __init__(self, fmt=None, datefmt=None):
+        # NOTE(jkoelker) we ignore the fmt argument, but its still there
+        #                since logging.config.fileConfig passes it.
+        self.datefmt = datefmt
+
+    def formatException(self, ei, strip_newlines=True):
+        lines = traceback.format_exception(*ei)
+        if strip_newlines:
+            lines = [itertools.ifilter(
+                lambda x: x,
+                line.rstrip().splitlines()) for line in lines]
+            lines = list(itertools.chain(*lines))
+        return lines
+
+    def format(self, record):
+        message = {'message': record.getMessage(),
+                   'asctime': self.formatTime(record, self.datefmt),
+                   'name': record.name,
+                   'msg': record.msg,
+                   'args': record.args,
+                   'levelname': record.levelname,
+                   'levelno': record.levelno,
+                   'pathname': record.pathname,
+                   'filename': record.filename,
+                   'module': record.module,
+                   'lineno': record.lineno,
+                   'funcname': record.funcName,
+                   'created': record.created,
+                   'msecs': record.msecs,
+                   'relative_created': record.relativeCreated,
+                   'thread': record.thread,
+                   'thread_name': record.threadName,
+                   'process_name': record.processName,
+                   'process': record.process,
+                   'traceback': None}
+
+        if hasattr(record, 'extra'):
+            message['extra'] = record.extra
+
+        if record.exc_info:
+            message['traceback'] = self.formatException(record.exc_info)
+
+        return jsonutils.dumps(message)
+
+
+def _create_logging_excepthook(product_name):
+    def logging_excepthook(type, value, tb):
+        extra = {}
+        if CONF.verbose:
+            extra['exc_info'] = (type, value, tb)
+        getLogger(product_name).critical(str(value), **extra)
+    return logging_excepthook
+
+
+class LogConfigError(Exception):
+
+    message = ('Error loading logging config %(log_config)s: %(err_msg)s')
+
+    def __init__(self, log_config, err_msg):
+        self.log_config = log_config
+        self.err_msg = err_msg
+
+    def __str__(self):
+        return self.message % dict(log_config=self.log_config,
+                                   err_msg=self.err_msg)
+
+
+def _load_log_config(log_config):
+    try:
+        logging.config.fileConfig(log_config)
+    except ConfigParser.Error as exc:
+        raise LogConfigError(log_config, str(exc))
+
+
+def setup(product_name):
+    """Setup logging."""
+    if CONF.log_config:
+        _load_log_config(CONF.log_config)
+    else:
+        _setup_logging_from_conf()
+    sys.excepthook = _create_logging_excepthook(product_name)
+
+
+def set_defaults(logging_context_format_string):
+    cfg.set_defaults(log_opts,
+                     logging_context_format_string=
+                     logging_context_format_string)
+
+
+def _find_facility_from_conf():
+    facility_names = logging.handlers.SysLogHandler.facility_names
+    facility = getattr(logging.handlers.SysLogHandler,
+                       CONF.syslog_log_facility,
+                       None)
+
+    if facility is None and CONF.syslog_log_facility in facility_names:
+        facility = facility_names.get(CONF.syslog_log_facility)
+
+    if facility is None:
+        valid_facilities = facility_names.keys()
+        consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
+                  'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
+                  'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
+                  'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
+                  'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
+        valid_facilities.extend(consts)
+        raise TypeError(('syslog facility must be one of: %s') %
+                        ', '.join("'%s'" % fac
+                                  for fac in valid_facilities))
+
+    return facility
+
+
+def _setup_logging_from_conf():
+    log_root = getLogger(None).logger
+    for handler in log_root.handlers:
+        log_root.removeHandler(handler)
+
+    if CONF.use_syslog:
+        facility = _find_facility_from_conf()
+        syslog = logging.handlers.SysLogHandler(address='/dev/log',
+                                                facility=facility)
+        log_root.addHandler(syslog)
+
+    logpath = _get_log_file_path()
+    if logpath:
+        filelog = logging.handlers.WatchedFileHandler(logpath)
+        log_root.addHandler(filelog)
+
+    if CONF.use_stderr:
+        streamlog = ColorHandler()
+        log_root.addHandler(streamlog)
+
+    elif not CONF.log_file:
+        # pass sys.stdout as a positional argument
+        # python2.6 calls the argument strm, in 2.7 it's stream
+        streamlog = logging.StreamHandler(sys.stdout)
+        log_root.addHandler(streamlog)
+
+    if CONF.publish_errors:
+        handler = importutils.import_object(
+            "quantum.openstack.common.log_handler.PublishErrorsHandler",
+            logging.ERROR)
+        log_root.addHandler(handler)
+
+    datefmt = CONF.log_date_format
+    for handler in log_root.handlers:
+        # NOTE(alaski): CONF.log_format overrides everything currently.  This
+        # should be deprecated in favor of context aware formatting.
+        if CONF.log_format:
+            handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
+                                                   datefmt=datefmt))
+            log_root.info('Deprecated: log_format is now deprecated and will '
+                          'be removed in the next release')
+        else:
+            handler.setFormatter(ContextFormatter(datefmt=datefmt))
+
+    if CONF.debug:
+        log_root.setLevel(logging.DEBUG)
+    elif CONF.verbose:
+        log_root.setLevel(logging.INFO)
+    else:
+        log_root.setLevel(logging.WARNING)
+
+    for pair in CONF.default_log_levels:
+        mod, _sep, level_name = pair.partition('=')
+        level = logging.getLevelName(level_name)
+        logger = logging.getLogger(mod)
+        logger.setLevel(level)
+
+_loggers = {}
+
+
+def getLogger(name='unknown', version='unknown'):
+    if name not in _loggers:
+        _loggers[name] = ContextAdapter(logging.getLogger(name),
+                                        name,
+                                        version)
+    return _loggers[name]
+
+
+def getLazyLogger(name='unknown', version='unknown'):
+    """Returns lazy logger.
+
+    Creates a pass-through logger that does not create the real logger
+    until it is really needed and delegates all calls to the real logger
+    once it is created.
+    """
+    return LazyAdapter(name, version)
+
+
+class WritableLogger(object):
+    """A thin wrapper that responds to `write` and logs."""
+
+    def __init__(self, logger, level=logging.INFO):
+        self.logger = logger
+        self.level = level
+
+    def write(self, msg):
+        self.logger.log(self.level, msg)
+
+
+class ContextFormatter(logging.Formatter):
+    """A context.RequestContext aware formatter configured through flags.
+
+    The flags used to set format strings are: logging_context_format_string
+    and logging_default_format_string.  You can also specify
+    logging_debug_format_suffix to append extra formatting if the log level is
+    debug.
+
+    For information about what variables are available for the formatter see:
+    http://docs.python.org/library/logging.html#formatter
+
+    """
+
+    def format(self, record):
+        """Uses contextstring if request_id is set, otherwise default."""
+        # NOTE(sdague): default the fancier formating params
+        # to an empty string so we don't throw an exception if
+        # they get used
+        for key in ('instance', 'color'):
+            if key not in record.__dict__:
+                record.__dict__[key] = ''
+
+        if record.__dict__.get('request_id', None):
+            self._fmt = CONF.logging_context_format_string
+        else:
+            self._fmt = CONF.logging_default_format_string
+
+        if (record.levelno == logging.DEBUG and
+                CONF.logging_debug_format_suffix):
+            self._fmt += " " + CONF.logging_debug_format_suffix
+
+        # Cache this on the record, Logger will respect our formated copy
+        if record.exc_info:
+            record.exc_text = self.formatException(record.exc_info, record)
+        return logging.Formatter.format(self, record)
+
+    def formatException(self, exc_info, record=None):
+        """Format exception output with CONF.logging_exception_prefix."""
+        if not record:
+            return logging.Formatter.formatException(self, exc_info)
+
+        stringbuffer = cStringIO.StringIO()
+        traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
+                                  None, stringbuffer)
+        lines = stringbuffer.getvalue().split('\n')
+        stringbuffer.close()
+
+        if CONF.logging_exception_prefix.find('%(asctime)') != -1:
+            record.asctime = self.formatTime(record, self.datefmt)
+
+        formatted_lines = []
+        for line in lines:
+            pl = CONF.logging_exception_prefix % record.__dict__
+            fl = '%s%s' % (pl, line)
+            formatted_lines.append(fl)
+        return '\n'.join(formatted_lines)
+
+
+class ColorHandler(logging.StreamHandler):
+    LEVEL_COLORS = {
+        logging.DEBUG: '\033[00;32m',  # GREEN
+        logging.INFO: '\033[00;36m',  # CYAN
+        logging.AUDIT: '\033[01;36m',  # BOLD CYAN
+        logging.WARN: '\033[01;33m',  # BOLD YELLOW
+        logging.ERROR: '\033[01;31m',  # BOLD RED
+        logging.CRITICAL: '\033[01;31m',  # BOLD RED
+    }
+
+    def format(self, record):
+        record.color = self.LEVEL_COLORS[record.levelno]
+        return logging.StreamHandler.format(self, record)
+
+
+class DeprecatedConfig(Exception):
+    message = ("Fatal call to deprecated config: %(msg)s")
+
+    def __init__(self, msg):
+        super(Exception, self).__init__(self.message % dict(msg=msg))

stackalytics/openstack/common/timeutils.py

@@ -0,0 +1,187 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Time related utilities and helper functions.
+"""
+
+import calendar
+import datetime
+
+import iso8601
+
+
+# ISO 8601 extended time format with microseconds
+_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
+_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
+PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
+
+
+def isotime(at=None, subsecond=False):
+    """Stringify time in ISO 8601 format."""
+    if not at:
+        at = utcnow()
+    st = at.strftime(_ISO8601_TIME_FORMAT
+                     if not subsecond
+                     else _ISO8601_TIME_FORMAT_SUBSECOND)
+    tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
+    st += ('Z' if tz == 'UTC' else tz)
+    return st
+
+
+def parse_isotime(timestr):
+    """Parse time from ISO 8601 format."""
+    try:
+        return iso8601.parse_date(timestr)
+    except iso8601.ParseError as e:
+        raise ValueError(e.message)
+    except TypeError as e:
+        raise ValueError(e.message)
+
+
+def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
+    """Returns formatted utcnow."""
+    if not at:
+        at = utcnow()
+    return at.strftime(fmt)
+
+
+def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
+    """Turn a formatted time back into a datetime."""
+    return datetime.datetime.strptime(timestr, fmt)
+
+
+def normalize_time(timestamp):
+    """Normalize time in arbitrary timezone to UTC naive object."""
+    offset = timestamp.utcoffset()
+    if offset is None:
+        return timestamp
+    return timestamp.replace(tzinfo=None) - offset
+
+
+def is_older_than(before, seconds):
+    """Return True if before is older than seconds."""
+    if isinstance(before, basestring):
+        before = parse_strtime(before).replace(tzinfo=None)
+    return utcnow() - before > datetime.timedelta(seconds=seconds)
+
+
+def is_newer_than(after, seconds):
+    """Return True if after is newer than seconds."""
+    if isinstance(after, basestring):
+        after = parse_strtime(after).replace(tzinfo=None)
+    return after - utcnow() > datetime.timedelta(seconds=seconds)
+
+
+def utcnow_ts():
+    """Timestamp version of our utcnow function."""
+    return calendar.timegm(utcnow().timetuple())
+
+
+def utcnow():
+    """Overridable version of utils.utcnow."""
+    if utcnow.override_time:
+        try:
+            return utcnow.override_time.pop(0)
+        except AttributeError:
+            return utcnow.override_time
+    return datetime.datetime.utcnow()
+
+
+def iso8601_from_timestamp(timestamp):
+    """Returns a iso8601 formated date from timestamp."""
+    return isotime(datetime.datetime.utcfromtimestamp(timestamp))
+
+
+utcnow.override_time = None
+
+
+def set_time_override(override_time=datetime.datetime.utcnow()):
+    """Overrides utils.utcnow.
+
+    Make it return a constant time or a list thereof, one at a time.
+    """
+    utcnow.override_time = override_time
+
+
+def advance_time_delta(timedelta):
+    """Advance overridden time using a datetime.timedelta."""
+    assert(not utcnow.override_time is None)
+    try:
+        for dt in utcnow.override_time:
+            dt += timedelta
+    except TypeError:
+        utcnow.override_time += timedelta
+
+
+def advance_time_seconds(seconds):
+    """Advance overridden time by seconds."""
+    advance_time_delta(datetime.timedelta(0, seconds))
+
+
+def clear_time_override():
+    """Remove the overridden time."""
+    utcnow.override_time = None
+
+
+def marshall_now(now=None):
+    """Make an rpc-safe datetime with microseconds.
+
+    Note: tzinfo is stripped, but not required for relative times.
+    """
+    if not now:
+        now = utcnow()
+    return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
+                minute=now.minute, second=now.second,
+                microsecond=now.microsecond)
+
+
+def unmarshall_time(tyme):
+    """Unmarshall a datetime dict."""
+    return datetime.datetime(day=tyme['day'],
+                             month=tyme['month'],
+                             year=tyme['year'],
+                             hour=tyme['hour'],
+                             minute=tyme['minute'],
+                             second=tyme['second'],
+                             microsecond=tyme['microsecond'])
+
+
+def delta_seconds(before, after):
+    """Return the difference between two timing objects.
+
+    Compute the difference in seconds between two date, time, or
+    datetime objects (as a float, to microsecond resolution).
+    """
+    delta = after - before
+    try:
+        return delta.total_seconds()
+    except AttributeError:
+        return ((delta.days * 24 * 3600) + delta.seconds +
+                float(delta.microseconds) / (10 ** 6))
+
+
+def is_soon(dt, window):
+    """Determines if time is going to happen in the next window seconds.
+
+    :params dt: the time
+    :params window: minimum seconds to remain to consider the time not soon
+
+    :return: True if expiration is within the given duration
+    """
+    soon = (utcnow() + datetime.timedelta(seconds=window))
+    return normalize_time(dt) <= soon

stackalytics/processor/__init__.py

@@ -0,0 +1 @@
+__author__ = 'ishakhat'

stackalytics/processor/commit_processor.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+from launchpadlib import launchpad
+from oslo.config import cfg
+
+LOG = logging.getLogger(__name__)
+
+
+COMMIT_PROCESSOR_DUMMY = 0
+COMMIT_PROCESSOR_CACHED = 1
+
+
+class CommitProcessor(object):
+    def __init__(self, persistent_storage):
+        self.persistent_storage = persistent_storage
+
+    def process(self, commit_iterator):
+        pass
+
+
+class DummyProcessor(CommitProcessor):
+    def __init__(self, persistent_storage):
+        super(DummyProcessor, self).__init__(persistent_storage)
+
+    def process(self, commit_iterator):
+        return commit_iterator
+
+
+class CachedProcessor(CommitProcessor):
+    def __init__(self, persistent_storage):
+        super(CachedProcessor, self).__init__(persistent_storage)
+
+        companies = persistent_storage.get_companies()
+        self.domains_index = {}
+        for company in companies:
+            for domain in company['domains']:
+                self.domains_index[domain] = company['company_name']
+
+        users = persistent_storage.get_users()
+        self.users_index = {}
+        for user in users:
+            for email in user['emails']:
+                self.users_index[email] = user
+
+        LOG.debug('Cached commit processor is instantiated')
+
+    def _find_company(self, companies, date):
+        for r in companies:
+            if date < r['end_date']:
+                return r['company_name']
+        return companies[-1]['company_name']
+
+    def _get_company_by_email(self, email):
+        name, at, domain = email.partition('@')
+        if domain:
+            parts = domain.split('.')
+            for i in range(len(parts), 1, -1):
+                m = '.'.join(parts[len(parts) - i:])
+                if m in self.domains_index:
+                    return self.domains_index[m]
+        return None
+
+    def _unknown_user_email(self, email):
+
+        lp_profile = None
+        if not re.match(r'[^@]+@[^@]+\.[^@]+', email):
+            LOG.debug('User email is not valid %s' % email)
+        else:
+            LOG.debug('Lookup user email %s at Launchpad' % email)
+            lp = launchpad.Launchpad.login_anonymously(cfg.CONF.launchpad_user)
+            try:
+                lp_profile = lp.people.getByEmail(email=email)
+            except Exception as error:
+                LOG.warn('Lookup of email %s failed %s' %
+                         (email, error.message))
+        if not lp_profile:
+            # user is not found in Launchpad, create dummy record for commit
+            # update
+            LOG.debug('Email is not found at Launchpad, mapping to nobody')
+            user = {
+                'launchpad_id': None,
+                'companies': [{
+                    'company_name': self.domains_index[''],
+                    'end_date': 0
+                }]
+            }
+        else:
+            # get user's launchpad id from his profile
+            launchpad_id = lp_profile.name
+            LOG.debug('Found user %s' % launchpad_id)
+
+            # check if user with launchpad_id exists in persistent storage
+            persistent_user_iterator = self.persistent_storage.get_users(
+                launchpad_id=launchpad_id)
+
+            for persistent_user in persistent_user_iterator:
+                break
+            else:
+                persistent_user = None
+
+            if persistent_user:
+                # user already exist, merge
+                LOG.debug('User exists in persistent storage, add new email')
+                persistent_user_email = persistent_user['emails'][0]
+                if persistent_user_email not in self.users_index:
+                    raise Exception('User index is not valid')
+                user = self.users_index[persistent_user_email]
+                user['emails'].append(email)
+                self.persistent_storage.update_user(user)
+            else:
+                # add new user
+                LOG.debug('Add new user into persistent storage')
+                company = (self._get_company_by_email(email) or
+                           self.domains_index[''])
+                user = {
+                    'launchpad_id': launchpad_id,
+                    'user_name': lp_profile.display_name,
+                    'emails': [email],
+                    'companies': [{
+                        'company_name': company,
+                        'end_date': 0,
+                    }],
+                }
+                self.persistent_storage.insert_user(user)
+
+        # update local index
+        self.users_index[email] = user
+        return user
+
+    def _update_commit_with_user_data(self, commit):
+        email = commit['author_email'].lower()
+        if email in self.users_index:
+            user = self.users_index[email]
+        else:
+            user = self._unknown_user_email(email)
+        commit['launchpad_id'] = user['launchpad_id']
+        company = self._get_company_by_email(email)
+        if not company:
+            company = self._find_company(user['companies'], commit['date'])
+        commit['company_name'] = company
+
+    def process(self, commit_iterator):
+
+        for commit in commit_iterator:
+            self._update_commit_with_user_data(commit)
+
+            yield commit
+
+
+class CommitProcessorFactory(object):
+    @staticmethod
+    def get_processor(commit_processor_type, persistent_storage):
+        LOG.debug('Factory is asked for commit processor type %s' %
+                  commit_processor_type)
+        if commit_processor_type == COMMIT_PROCESSOR_DUMMY:
+            return DummyProcessor(persistent_storage)
+        elif commit_processor_type == COMMIT_PROCESSOR_CACHED:
+            return CachedProcessor(persistent_storage)
+        else:
+            raise Exception('Unknown commit processor type %s' %
+                            commit_processor_type)

stackalytics/processor/main.py

@@ -0,0 +1,164 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+from oslo.config import cfg
+import psutil
+from psutil import _error
+import sh
+
+from stackalytics.openstack.common import log as logging
+from stackalytics.openstack.common.timeutils import iso8601_from_timestamp
+from stackalytics.processor import commit_processor
+from stackalytics.processor.persistent_storage import PersistentStorageFactory
+from stackalytics.processor.runtime_storage import RuntimeStorageFactory
+from stackalytics.processor.vcs import VcsFactory
+
+
+LOG = logging.getLogger(__name__)
+
+OPTS = [
+    cfg.StrOpt('default-data', default='etc/default_data.json',
+               help='Default data'),
+    cfg.StrOpt('sources-root', default=None, required=True,
+               help='The folder that holds all project sources to analyze'),
+    cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211',
+               help='Storage URI'),
+    cfg.StrOpt('frontend-update-address',
+               default='http://user:user@localhost/update/%s',
+               help='Address of update handler'),
+    cfg.StrOpt('repo-poll-period', default='300',
+               help='Repo poll period in seconds'),
+    cfg.StrOpt('persistent-storage-uri', default='mongodb://localhost',
+               help='URI of persistent storage'),
+    cfg.BoolOpt('sync-default-data', default=False,
+                help='Update persistent storage with default data. '
+                     'Existing data is not overwritten'),
+    cfg.BoolOpt('force-sync-default-data', default=False,
+                help='Completely overwrite persistent storage with the '
+                     'default data'),
+    cfg.StrOpt('launchpad-user', default='stackalytics-bot',
+               help='User to access Launchpad'),
+]
+
+
+def get_pids():
+    uwsgi_dict = {}
+    for pid in psutil.get_pid_list():
+        try:
+            p = psutil.Process(pid)
+            if p.cmdline and p.cmdline[0].find('/uwsgi '):
+                uwsgi_dict[p.pid] = p.parent
+        except _error.NoSuchProcess:
+            # the process may disappear after get_pid_list call, ignore it
+            pass
+
+    result = set()
+    for pid in uwsgi_dict:
+        if uwsgi_dict[pid] in uwsgi_dict:
+            result.add(pid)
+
+    return result
+
+
+def update_pid(pid):
+    url = cfg.CONF.frontend_update_address % pid
+    sh.curl(url)
+
+
+def update_pids(runtime_storage):
+    pids = get_pids()
+    if not pids:
+        return
+    runtime_storage.active_pids(pids)
+    current_time = time.time()
+    for pid in pids:
+        if current_time > runtime_storage.get_pid_update_time(pid):
+            update_pid(pid)
+    return current_time
+
+
+def process_repo(repo, runtime_storage, processor):
+    uri = repo['uri']
+    LOG.debug('Processing repo uri %s' % uri)
+
+    vcs = VcsFactory.get_vcs(repo)
+    vcs.fetch()
+
+    for branch in repo['branches']:
+        LOG.debug('Processing repo %s, branch %s' % (uri, branch))
+
+        head_commit_id = runtime_storage.get_head_commit_id(uri, branch)
+
+        commit_iterator = vcs.log(branch, head_commit_id)
+        processed_commit_iterator = processor.process(commit_iterator)
+        runtime_storage.set_records(processed_commit_iterator)
+
+        head_commit_id = vcs.get_head_commit_id(branch)
+        runtime_storage.set_head_commit_id(uri, branch, head_commit_id)
+
+
+def update_repos(runtime_storage, persistent_storage):
+    current_time = time.time()
+    repo_update_time = runtime_storage.get_repo_update_time()
+
+    if current_time < repo_update_time:
+        LOG.info('The next update is scheduled at %s. Skipping' %
+                 iso8601_from_timestamp(repo_update_time))
+        return
+
+    repos = persistent_storage.get_repos()
+    processor = commit_processor.CommitProcessorFactory.get_processor(
+        commit_processor.COMMIT_PROCESSOR_CACHED,
+        persistent_storage)
+
+    for repo in repos:
+        process_repo(repo, runtime_storage, processor)
+
+    runtime_storage.set_repo_update_time(time.time() +
+                                         int(cfg.CONF.repo_poll_period))
+
+
+def main():
+    # init conf and logging
+    conf = cfg.CONF
+    conf.register_cli_opts(OPTS)
+    conf.register_opts(OPTS)
+    conf()
+
+    logging.setup('stackalytics')
+    LOG.info('Logging enabled')
+
+    persistent_storage = PersistentStorageFactory.get_storage(
+        cfg.CONF.persistent_storage_uri)
+
+    if conf.sync_default_data or conf.force_sync_default_data:
+        LOG.info('Going to synchronize persistent storage with default data '
+                 'from file %s' % cfg.CONF.default_data)
+        persistent_storage.sync(cfg.CONF.default_data,
+                                force=conf.force_sync_default_data)
+        return 0
+
+    runtime_storage = RuntimeStorageFactory.get_storage(
+        cfg.CONF.runtime_storage_uri)
+
+    update_pids(runtime_storage)
+
+    update_repos(runtime_storage, persistent_storage)
+
+
+if __name__ == '__main__':
+    main()

stackalytics/processor/persistent_storage.py

@@ -0,0 +1,150 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import re
+
+import pymongo
+
+from stackalytics.processor.user_utils import normalize_user
+
+LOG = logging.getLogger(__name__)
+
+
+class PersistentStorage(object):
+    def __init__(self, uri):
+        pass
+
+    def sync(self, default_data_file_name, force=False):
+        if force:
+            self.clean_all()
+
+        default_data = self._read_default_persistent_storage(
+            default_data_file_name)
+
+        self._build_index(default_data['repos'], 'uri',
+                          self.get_repos, self.insert_repo)
+        self._build_index(default_data['companies'], 'company_name',
+                          self.get_companies, self.insert_company)
+        self._build_index(default_data['users'], 'launchpad_id',
+                          self.get_users, self.insert_user)
+        self._build_index(default_data['releases'], 'release_name',
+                          self.get_releases, self.insert_release)
+
+    def _build_index(self, default_data, primary_key, getter, inserter):
+        # loads all items from persistent storage
+        existing_items = set([item[primary_key] for item in getter()])
+        # inserts items from default storage that are not in persistent storage
+        map(inserter, [item for item in default_data
+                       if item[primary_key] not in existing_items])
+
+    def get_companies(self, **criteria):
+        pass
+
+    def insert_company(self, company):
+        pass
+
+    def get_repos(self, **criteria):
+        pass
+
+    def insert_repo(self, repo):
+        pass
+
+    def get_users(self, **criteria):
+        pass
+
+    def insert_user(self, user):
+        pass
+
+    def update_user(self, user):
+        pass
+
+    def get_releases(self, **criteria):
+        pass
+
+    def insert_release(self, release):
+        pass
+
+    def clean_all(self):
+        pass
+
+    def _read_default_persistent_storage(self, file_name):
+        try:
+            with open(file_name, 'r') as content_file:
+                content = content_file.read()
+                return json.loads(content)
+        except Exception as e:
+            LOG.error('Error while reading config: %s' % e)
+
+
+class MongodbStorage(PersistentStorage):
+    def __init__(self, uri):
+        super(MongodbStorage, self).__init__(uri)
+
+        self.client = pymongo.MongoClient(uri)
+        self.mongo = self.client.stackalytics
+
+        self.mongo.companies.create_index([("company", pymongo.ASCENDING)])
+        self.mongo.repos.create_index([("uri", pymongo.ASCENDING)])
+        self.mongo.users.create_index([("launchpad_id", pymongo.ASCENDING)])
+        self.mongo.releases.create_index([("releases", pymongo.ASCENDING)])
+
+        LOG.debug('Mongodb storage is created')
+
+    def clean_all(self):
+        LOG.debug('Clear all tables')
+        self.mongo.companies.remove()
+        self.mongo.repos.remove()
+        self.mongo.users.remove()
+        self.mongo.releases.remove()
+
+    def get_companies(self, **criteria):
+        return self.mongo.companies.find(criteria)
+
+    def insert_company(self, company):
+        self.mongo.companies.insert(company)
+
+    def get_repos(self, **criteria):
+        return self.mongo.repos.find(criteria)
+
+    def insert_repo(self, repo):
+        self.mongo.repos.insert(repo)
+
+    def get_users(self, **criteria):
+        return self.mongo.users.find(criteria)
+
+    def insert_user(self, user):
+        self.mongo.users.insert(normalize_user(user))
+
+    def update_user(self, user):
+        normalize_user(user)
+        launchpad_id = user['launchpad_id']
+        self.mongo.users.update({'launchpad_id': launchpad_id}, user)
+
+    def get_releases(self, **criteria):
+        return self.mongo.releases.find(criteria)
+
+    def insert_release(self, release):
+        self.mongo.releases.insert(release)
+
+
+class PersistentStorageFactory(object):
+    @staticmethod
+    def get_storage(uri):
+        LOG.debug('Persistent storage is requested for uri %s' % uri)
+        match = re.search(r'^mongodb:\/\/', uri)
+        if match:
+            return MongodbStorage(uri)

stackalytics/processor/runtime_storage.py

@@ -0,0 +1,196 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import re
+import urllib
+
+import memcache
+from oslo.config import cfg
+
+LOG = logging.getLogger(__name__)
+
+
+class RuntimeStorage(object):
+    def __init__(self, uri):
+        pass
+
+    def set_records(self, records_iterator):
+        pass
+
+    def get_head_commit_id(self, uri, branch):
+        pass
+
+    def set_head_commit_id(self, uri, branch, head_commit_id):
+        pass
+
+    def get_update(self, pid):
+        pass
+
+    def active_pids(self, pids):
+        pass
+
+    def get_pid_update_time(self, pid):
+        pass
+
+    def set_pid_update_time(self, pid, time):
+        pass
+
+    def get_repo_update_time(self):
+        pass
+
+    def set_repo_update_time(self, time):
+        pass
+
+
+class MemcachedStorage(RuntimeStorage):
+
+    def __init__(self, uri):
+        super(MemcachedStorage, self).__init__(uri)
+
+        stripped = re.sub(r'memcached:\/\/', '', uri)
+        if stripped:
+            storage_uri = stripped.split(',')
+            self.memcached = memcache.Client(storage_uri)
+            self._build_index()
+        else:
+            raise Exception('Invalid storage uri %s' % cfg.CONF.storage_uri)
+
+    def set_records(self, records_iterator):
+        for record in records_iterator:
+            if record['commit_id'] in self.commit_id_index:
+                # update
+                record_id = self.commit_id_index[record['commit_id']]
+                old_record = self.memcached.get(
+                    self._get_record_name(record_id))
+                old_record['branches'] |= record['branches']
+                LOG.debug('Update record %s' % record)
+                self.memcached.set(self._get_record_name(record_id),
+                                   old_record)
+            else:
+                # insert record
+                record_id = self._get_record_count()
+                record['record_id'] = record_id
+                LOG.debug('Insert new record %s' % record)
+                self.memcached.set(self._get_record_name(record_id), record)
+                self._set_record_count(record_id + 1)
+
+            self._commit_update(record_id)
+
+    def get_head_commit_id(self, uri, branch):
+        key = str(urllib.quote_plus(uri) + ':' + branch)
+        return self.memcached.get(key)
+
+    def set_head_commit_id(self, uri, branch, head_commit_id):
+        key = str(urllib.quote_plus(uri) + ':' + branch)
+        self.memcached.set(key, head_commit_id)
+
+    def get_update(self, pid):
+        last_update = self.memcached.get('pid:%s' % pid)
+        update_count = self._get_update_count()
+
+        self.memcached.set('pid:%s' % pid, update_count)
+        self._set_pids(pid)
+
+        if not last_update:
+            for record_id in range(0, self._get_record_count()):
+                yield self.memcached.get(self._get_record_name(record_id))
+        else:
+            for update_id in range(last_update, update_count):
+                yield self.memcached.get(self._get_record_name(
+                    self.memcached.get('update:%s' % update_id)))
+
+    def active_pids(self, pids):
+        stored_pids = self.memcached.get('pids') or set()
+        for pid in stored_pids:
+            if pid not in pids:
+                self.memcached.delete('pid:%s' % pid)
+                self.memcached.delete('pid_update_time:%s' % pid)
+
+        self.memcached.set('pids', pids)
+
+        # remove unneeded updates
+        min_update = self._get_update_count()
+        for pid in pids:
+            n = self.memcached.get('pid:%s' % pid)
+            if n:
+                if n < min_update:
+                    min_update = n
+
+        first_valid_update_id = self.memcached.get('first_valid_update_id')
+        if not first_valid_update_id:
+            first_valid_update_id = 0
+
+        for i in range(first_valid_update_id, min_update):
+            self.memcached.delete('update:%s' % i)
+
+        self.memcached.set('first_valid_update_id', min_update)
+
+    def get_pid_update_time(self, pid):
+        return self.memcached.get('pid_update_time:%s' % pid) or 0
+
+    def set_pid_update_time(self, pid, time):
+        self.memcached.set('pid_update_time:%s' % pid, time)
+
+    def get_repo_update_time(self):
+        return self.memcached.get('repo_update_time') or 0
+
+    def set_repo_update_time(self, time):
+        self.memcached.set('repo_update_time', time)
+
+    def _get_update_count(self):
+        return self.memcached.get('update:count') or 0
+
+    def _set_pids(self, pid):
+        pids = self.memcached.get('pids') or set()
+        if pid in pids:
+            return
+        pids.add(pid)
+        self.memcached.set('pids', pids)
+
+    def _get_record_name(self, record_id):
+        return 'record:%s' % record_id
+
+    def _get_record_count(self):
+        return self.memcached.get('record:count') or 0
+
+    def _set_record_count(self, count):
+        self.memcached.set('record:count', count)
+
+    def _get_all_records(self):
+        count = self.memcached.get('record:count') or 0
+        for i in range(0, count):
+            yield self.memcached.get('record:%s' % i)
+
+    def _commit_update(self, record_id):
+        count = self._get_update_count()
+        self.memcached.set('update:%s' % count, record_id)
+        self.memcached.set('update:count', count + 1)
+
+    def _build_index(self):
+        self.commit_id_index = {}
+        for record in self._get_all_records():
+            self.commit_id_index[record['commit_id']] = record['record_id']
+
+
+class RuntimeStorageFactory(object):
+
+    @staticmethod
+    def get_storage(uri):
+        LOG.debug('Runtime storage is requested for uri %s' % uri)
+        match = re.search(r'^memcached:\/\/', uri)
+        if match:
+            return MemcachedStorage(uri)

stackalytics/processor/user_utils.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import time
+
+
+def normalize_user(user):
+    user['emails'] = [email.lower() for email in user['emails']]
+    user['launchpad_id'] = user['launchpad_id'].lower()
+
+    for c in user['companies']:
+        end_date_numeric = 0
+        if c['end_date']:
+            end_date_numeric = date_to_timestamp(c['end_date'])
+        c['end_date'] = end_date_numeric
+
+    # sort companies by end_date
+    def end_date_comparator(x, y):
+        if x["end_date"] == 0:
+            return 1
+        elif y["end_date"] == 0:
+            return -1
+        else:
+            return cmp(x["end_date"], y["end_date"])
+
+    user['companies'].sort(cmp=end_date_comparator)
+    return user
+
+
+def date_to_timestamp(d):
+    if d == 'now':
+        return int(time.time())
+    return int(time.mktime(
+        datetime.datetime.strptime(d, '%Y-%b-%d').timetuple()))
+
+
+def timestamp_to_week(timestamp):
+    # Jan 4th 1970 is the first Sunday in the Epoch
+    return (timestamp - 3 * 24 * 3600) // (7 * 24 * 3600)
+
+
+def week_to_date(week):
+    timestamp = week * 7 * 24 * 3600 + 3 * 24 * 3600
+    return (datetime.datetime.fromtimestamp(timestamp).
+            strftime('%Y-%m-%d %H:%M:%S'))

stackalytics/processor/vcs.py

@@ -0,0 +1,175 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+import os
+import re
+
+from oslo.config import cfg
+import sh
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Vcs(object):
+    def __init__(self, repo):
+        self.repo = repo
+
+    def fetch(self):
+        pass
+
+    def log(self, branch, head_commit_id):
+        pass
+
+    def get_head_commit_id(self, branch):
+        pass
+
+
+GIT_LOG_PARAMS = [
+    ('commit_id', '%H'),
+    ('date', '%at'),
+    ('author', '%an'),
+    ('author_email', '%ae'),
+    ('author_email', '%ae'),
+    ('subject', '%s'),
+    ('message', '%b'),
+]
+GIT_LOG_FORMAT = ''.join([(r[0] + ':' + r[1] + '%n')
+                          for r in GIT_LOG_PARAMS]) + 'diff_stat:'
+DIFF_STAT_PATTERN = ('[^\d]+(\d+)\s+[^\s]*\s+changed'
+                     '(,\s+(\d+)\s+([^\d\s]*)\s+(\d+)?)?')
+GIT_LOG_PATTERN = re.compile(''.join([(r[0] + ':(.*?)\n')
+                                      for r in GIT_LOG_PARAMS]) +
+                             'diff_stat:' + DIFF_STAT_PATTERN,
+                             re.DOTALL)
+
+MESSAGE_PATTERNS = {
+    'bug_id': re.compile('bug\s+#?([\d]{5,7})', re.IGNORECASE),
+    'blueprint_id': re.compile('blueprint\s+([\w-]{6,})', re.IGNORECASE),
+    'change_id': re.compile('Change-Id: (I[0-9a-f]{40})', re.IGNORECASE),
+}
+
+
+class Git(Vcs):
+
+    def __init__(self, repo):
+        super(Git, self).__init__(repo)
+        uri = self.repo['uri']
+        match = re.search(r'([^\/]+)\.git$', uri)
+        if match:
+            self.module = match.group(1)
+        else:
+            raise Exception('Unexpected uri %s for git' % uri)
+        self.release_index = {}
+
+    def _chdir(self):
+        folder = os.path.normpath(cfg.CONF.sources_root + '/' + self.module)
+        os.chdir(folder)
+
+    def fetch(self):
+        LOG.debug('Fetching repo uri %s' % self.repo['uri'])
+
+        folder = os.path.normpath(cfg.CONF.sources_root + '/' + self.module)
+
+        if not os.path.exists(folder):
+            os.chdir(cfg.CONF.sources_root)
+            sh.git('clone', '%s' % self.repo['uri'])
+            os.chdir(folder)
+        else:
+            self._chdir()
+            sh.git('pull', 'origin')
+
+        for release in self.repo['releases']:
+            release_name = release['release_name'].lower()
+            tag_range = release['tag_from'] + '..' + release['tag_to']
+            git_log_iterator = sh.git('log', '--pretty=%H', tag_range,
+                                      _tty_out=False)
+            for commit_id in git_log_iterator:
+                self.release_index[commit_id.strip()] = release_name
+
+    def log(self, branch, head_commit_id):
+        LOG.debug('Parsing git log for repo uri %s' % self.repo['uri'])
+
+        self._chdir()
+        sh.git('checkout', '%s' % branch)
+        commit_range = 'HEAD'
+        if head_commit_id:
+            commit_range = head_commit_id + '..HEAD'
+        output = sh.git('log', '--pretty=%s' % GIT_LOG_FORMAT, '--shortstat',
+                        '-M', '--no-merges', commit_range, _tty_out=False)
+
+        for rec in re.finditer(GIT_LOG_PATTERN, str(output)):
+            i = 1
+            commit = {}
+            for param in GIT_LOG_PARAMS:
+                commit[param[0]] = unicode(rec.group(i), 'utf8')
+                i += 1
+
+            commit['files_changed'] = int(rec.group(i))
+            i += 1
+            lines_changed_group = rec.group(i)
+            i += 1
+            lines_changed = rec.group(i)
+            i += 1
+            deleted_or_inserted = rec.group(i)
+            i += 1
+            lines_deleted = rec.group(i)
+            i += 1
+
+            if lines_changed_group:  # there inserted or deleted lines
+                if not lines_deleted:
+                    if deleted_or_inserted[0] == 'd':  # deleted
+                        lines_deleted = lines_changed
+                        lines_changed = 0
+
+            commit['lines_added'] = int(lines_changed or 0)
+            commit['lines_deleted'] = int(lines_deleted or 0)
+
+            for key in MESSAGE_PATTERNS:
+                match = re.search(MESSAGE_PATTERNS[key], commit['message'])
+                if match:
+                    commit[key] = match.group(1)
+                else:
+                    commit[key] = None
+
+            commit['date'] = int(commit['date'])
+            commit['module'] = self.module
+            commit['branches'] = set([branch])
+            if commit['commit_id'] in self.release_index:
+                commit['release'] = self.release_index[commit['commit_id']]
+            else:
+                commit['release'] = None
+
+            yield commit
+
+    def get_head_commit_id(self, branch):
+        LOG.debug('Get head commit for repo uri %s' % self.repo['uri'])
+
+        self._chdir()
+        sh.git('checkout', '%s' % branch)
+        return str(sh.git('rev-parse', 'HEAD')).strip()
+
+
+class VcsFactory(object):
+
+    @staticmethod
+    def get_vcs(repo):
+        uri = repo['uri']
+        LOG.debug('Factory is asked for Vcs uri %s' % uri)
+        match = re.search(r'\.git$', uri)
+        if match:
+            return Git(repo)
+            #todo others vcs to be implemented

test-requirements.txt

@@ -7,7 +7,7 @@ hacking>=0.5.3,<0.6
 coverage
 discover
 fixtures>=0.3.12
-mox
+mock
 python-subunit
 testrepository>=0.0.13
 testtools>=0.9.22

tests/unit/test_commit_processor.py

@@ -0,0 +1,231 @@
+from launchpadlib import launchpad
+import mock
+from oslo.config import cfg
+import testtools
+
+from stackalytics.processor import commit_processor
+from stackalytics.processor import persistent_storage
+
+
+class TestCommitProcessor(testtools.TestCase):
+    def setUp(self):
+        super(TestCommitProcessor, self).setUp()
+
+        p_storage = mock.Mock(persistent_storage.PersistentStorage)
+        p_storage.get_companies = mock.Mock(return_value=[
+            {
+                'company_name': 'SuperCompany',
+                'domains': ['super.com', 'super.no']
+            },
+            {
+                "domains": ["nec.com", "nec.co.jp"],
+                "company_name": "NEC"
+            },
+            {
+                'company_name': '*independent',
+                'domains': ['']
+            },
+        ])
+        self.user = {
+            'launchpad_id': 'john_doe', 'user_name': 'John Doe',
+            'emails': ['johndoe@gmail.com', 'jdoe@super.no'],
+            'companies': [
+                {'company_name': '*independent',
+                 'end_date': 1234567890},
+                {'company_name': 'SuperCompany',
+                 'end_date': 0},
+            ]
+        }
+        p_storage.get_users = mock.Mock(return_value=[
+            self.user,
+        ])
+
+        self.persistent_storage = p_storage
+        self.commit_processor = commit_processor.CachedProcessor(p_storage)
+        self.launchpad_patch = mock.patch('launchpadlib.launchpad.Launchpad')
+        self.launchpad_patch.start()
+        cfg.CONF = mock.MagicMock()
+
+    def tearDown(self):
+        super(TestCommitProcessor, self).tearDown()
+        self.launchpad_patch.stop()
+
+    def test_get_company_by_email_mapped(self):
+        email = 'jdoe@super.no'
+        res = self.commit_processor._get_company_by_email(email)
+        self.assertEquals('SuperCompany', res)
+
+    def test_get_company_by_email_with_long_suffix_mapped(self):
+        email = 'man@mxw.nes.nec.co.jp'
+        res = self.commit_processor._get_company_by_email(email)
+        self.assertEquals('NEC', res)
+
+    def test_get_company_by_email_with_long_suffix_mapped_2(self):
+        email = 'man@mxw.nes.nec.com'
+        res = self.commit_processor._get_company_by_email(email)
+        self.assertEquals('NEC', res)
+
+    def test_get_company_by_email_not_mapped(self):
+        email = 'foo@boo.com'
+        res = self.commit_processor._get_company_by_email(email)
+        self.assertEquals(None, res)
+
+    def test_update_commit_existing_user(self):
+        commit = {
+            'author_email': 'johndoe@gmail.com',
+            'date': 1999999999,
+        }
+        self.commit_processor._update_commit_with_user_data(commit)
+
+        self.assertEquals('SuperCompany', commit['company_name'])
+        self.assertEquals('john_doe', commit['launchpad_id'])
+
+    def test_update_commit_existing_user_old_job(self):
+        commit = {
+            'author_email': 'johndoe@gmail.com',
+            'date': 1000000000,
+        }
+        self.commit_processor._update_commit_with_user_data(commit)
+
+        self.assertEquals('*independent', commit['company_name'])
+        self.assertEquals('john_doe', commit['launchpad_id'])
+
+    def test_update_commit_existing_user_new_email_known_company(self):
+        """
+        User is known to LP, his email is new to us, and maps to other company
+        Should return other company instead of those mentioned in user db
+        """
+        email = 'johndoe@nec.co.jp'
+        commit = {
+            'author_email': email,
+            'date': 1999999999,
+        }
+        lp_mock = mock.MagicMock()
+        launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
+        lp_profile = mock.Mock()
+        lp_profile.name = 'john_doe'
+        lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
+        user = self.user.copy()
+        # tell storage to return existing user
+        self.persistent_storage.get_users.return_value = [user]
+
+        self.commit_processor._update_commit_with_user_data(commit)
+
+        self.persistent_storage.update_user.assert_called_once_with(user)
+        lp_mock.people.getByEmail.assert_called_once_with(email=email)
+        self.assertIn(email, user['emails'])
+        self.assertEquals('NEC', commit['company_name'])
+        self.assertEquals('john_doe', commit['launchpad_id'])
+
+    def test_update_commit_existing_user_new_email_unknown_company(self):
+        """
+        User is known to LP, but his email is new to us. Should match
+        the user and return current company
+        """
+        email = 'johndoe@yahoo.com'
+        commit = {
+            'author_email': email,
+            'date': 1999999999,
+        }
+        lp_mock = mock.MagicMock()
+        launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
+        lp_profile = mock.Mock()
+        lp_profile.name = 'john_doe'
+        lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
+        user = self.user.copy()
+        # tell storage to return existing user
+        self.persistent_storage.get_users.return_value = [user]
+
+        self.commit_processor._update_commit_with_user_data(commit)
+
+        self.persistent_storage.update_user.assert_called_once_with(user)
+        lp_mock.people.getByEmail.assert_called_once_with(email=email)
+        self.assertIn(email, user['emails'])
+        self.assertEquals('SuperCompany', commit['company_name'])
+        self.assertEquals('john_doe', commit['launchpad_id'])
+
+    def test_update_commit_new_user(self):
+        """
+        User is known to LP, but new to us
+        Should add new user and set company depending on email
+        """
+        email = 'smith@nec.com'
+        commit = {
+            'author_email': email,
+            'date': 1999999999,
+        }
+        lp_mock = mock.MagicMock()
+        launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
+        lp_profile = mock.Mock()
+        lp_profile.name = 'smith'
+        lp_profile.display_name = 'Smith'
+        lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
+        self.persistent_storage.get_users.return_value = []
+
+        self.commit_processor._update_commit_with_user_data(commit)
+
+        lp_mock.people.getByEmail.assert_called_once_with(email=email)
+        self.assertEquals('NEC', commit['company_name'])
+        self.assertEquals('smith', commit['launchpad_id'])
+
+    def test_update_commit_new_user_unknown_to_lb(self):
+        """
+        User is new to us and not known to LP
+        Should set user name and empty LPid
+        """
+        email = 'inkognito@avs.com'
+        commit = {
+            'author_email': email,
+            'date': 1999999999,
+        }
+        lp_mock = mock.MagicMock()
+        launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
+        lp_mock.people.getByEmail = mock.Mock(return_value=None)
+        self.persistent_storage.get_users.return_value = []
+
+        self.commit_processor._update_commit_with_user_data(commit)
+
+        lp_mock.people.getByEmail.assert_called_once_with(email=email)
+        self.assertEquals('*independent', commit['company_name'])
+        self.assertEquals(None, commit['launchpad_id'])
+
+    def test_update_commit_new_user_lb_raises_error(self):
+        """
+        LP raises error during getting user info
+        """
+        email = 'smith@avs.com'
+        commit = {
+            'author_email': email,
+            'date': 1999999999,
+        }
+        lp_mock = mock.MagicMock()
+        launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
+        lp_mock.people.getByEmail = mock.Mock(return_value=None,
+                                              side_effect=Exception)
+        self.persistent_storage.get_users.return_value = []
+
+        self.commit_processor._update_commit_with_user_data(commit)
+
+        lp_mock.people.getByEmail.assert_called_once_with(email=email)
+        self.assertEquals('*independent', commit['company_name'])
+        self.assertEquals(None, commit['launchpad_id'])
+
+    def test_update_commit_invalid_email(self):
+        """
+        User's email is malformed
+        """
+        email = 'error.root'
+        commit = {
+            'author_email': email,
+            'date': 1999999999,
+        }
+        lp_mock = mock.MagicMock()
+        launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
+        lp_mock.people.getByEmail = mock.Mock(return_value=None)
+        self.persistent_storage.get_users.return_value = []
+
+        self.commit_processor._update_commit_with_user_data(commit)
+
+        self.assertEquals(0, lp_mock.people.getByEmail.called)
+        self.assertEquals('*independent', commit['company_name'])
+        self.assertEquals(None, commit['launchpad_id'])

tests/unit/test_dashboard.py

@@ -1,22 +0,0 @@
-import os
-import tempfile
-import unittest
-
-from dashboard import dashboard
-
-
-class DashboardTestCase(unittest.TestCase):
-
-    def setUp(self):
-        self.db_fd, dashboard.app.config['DATABASE'] = tempfile.mkstemp()
-        dashboard.app.config['TESTING'] = True
-        self.app = dashboard.app.test_client()
-        # dashboard.init_db()
-
-    def tearDown(self):
-        os.close(self.db_fd)
-        os.unlink(dashboard.app.config['DATABASE'])
-
-    def test_home_page(self):
-        rv = self.app.get('/')
-        assert rv.status_code == 200

tests/unit/test_vcs.py

@@ -0,0 +1,110 @@
+import mock
+import os
+import testtools
+
+from oslo.config import cfg
+
+from stackalytics.processor import vcs
+
+
+class TestVcsProcessor(testtools.TestCase):
+    def setUp(self):
+        super(TestVcsProcessor, self).setUp()
+
+        self.repo = {
+            'uri': 'git://github.com/dummy.git',
+            'releases': []
+        }
+        self.git = vcs.Git(self.repo)
+        cfg.CONF.sources_root = ''
+        os.chdir = mock.Mock()
+
+    def test_git_log(self):
+        with mock.patch('sh.git') as git_mock:
+            git_mock.return_value = '''
+commit_id:b5a416ac344160512f95751ae16e6612aefd4a57
+date:1369119386
+author:Akihiro MOTOKI
+author_email:motoki@da.jp.nec.com
+author_email:motoki@da.jp.nec.com
+subject:Remove class-based import in the code repo
+message:Fixes bug 1167901
+
+This commit also removes backslashes for line break.
+
+Change-Id: Id26fdfd2af4862652d7270aec132d40662efeb96
+
+diff_stat:
+
+ 21 files changed, 340 insertions(+), 408 deletions(-)
+commit_id:5be031f81f76d68c6e4cbaad2247044aca179843
+date:1370975889
+author:Monty Taylor
+author_email:mordred@inaugust.com
+author_email:mordred@inaugust.com
+subject:Remove explicit distribute depend.
+message:Causes issues with the recent re-merge with setuptools. Advice from
+upstream is to stop doing explicit depends.
+
+Change-Id: I70638f239794e78ba049c60d2001190910a89c90
+
+diff_stat:
+
+ 1 file changed, 1 deletion(-)
+commit_id:92811c76f3a8308b36f81e61451ec17d227b453b
+date:1369831203
+author:Mark McClain
+author_email:mark.mcclain@dreamhost.com
+author_email:mark.mcclain@dreamhost.com
+subject:add readme for 2.2.2
+message:Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
+
+diff_stat:
+
+ 1 file changed, 8 insertions(+)
+commit_id:92811c76f3a8308b36f81e61451ec17d227b453b
+date:1369831203
+author:John Doe
+author_email:john.doe@dreamhost.com
+author_email:john.doe@dreamhost.com
+subject:add readme for 2.2.2
+message:Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
+
+diff_stat:
+
+ 0 files changed
+commit_id:92811c76f3a8308b36f81e61451ec17d227b453b
+date:1369831203
+author:Doug Hoffner
+author_email:mark.mcclain@dreamhost.com
+author_email:mark.mcclain@dreamhost.com
+subject:add readme for 2.2.2
+message:Change-Id: Id32a4a72ec1d13992b306c4a38e73605758e26c7
+
+diff_stat:
+
+ 0 files changed, 0 insertions(+), 0 deletions(-)
+            '''
+
+        commits = list(self.git.log('dummy', 'dummy'))
+        self.assertEquals(5, len(commits))
+
+        self.assertEquals(21, commits[0]['files_changed'])
+        self.assertEquals(340, commits[0]['lines_added'])
+        self.assertEquals(408, commits[0]['lines_deleted'])
+
+        self.assertEquals(1, commits[1]['files_changed'])
+        self.assertEquals(0, commits[1]['lines_added'])
+        self.assertEquals(1, commits[1]['lines_deleted'])
+
+        self.assertEquals(1, commits[2]['files_changed'])
+        self.assertEquals(8, commits[2]['lines_added'])
+        self.assertEquals(0, commits[2]['lines_deleted'])
+
+        self.assertEquals(0, commits[3]['files_changed'])
+        self.assertEquals(0, commits[3]['lines_added'])
+        self.assertEquals(0, commits[3]['lines_deleted'])
+
+        self.assertEquals(0, commits[4]['files_changed'])
+        self.assertEquals(0, commits[4]['lines_added'])
+        self.assertEquals(0, commits[4]['lines_deleted'])

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,pep8
+envlist = py26,py27,pep8
 
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
@@ -26,7 +26,8 @@ downloadcache = ~/cache/pip
 
 [flake8]
 # E125 continuation line does not distinguish itself from next logical line
-ignore = E125
+# H404 multi line docstring should start with a summary
+ignore = E125,H404
 show-source = true
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,pycvsanaly2