@@ -274,7 +265,7 @@
{% endif %}
{% if show_module_activity %}
-
+
diff --git a/dashboard/web.py b/dashboard/web.py
index 2a1e689f9..98694be5f 100644
--- a/dashboard/web.py
+++ b/dashboard/web.py
@@ -49,8 +49,8 @@ METRIC_LABELS = {
'commits': 'Commits',
'marks': 'Reviews',
'emails': 'Emails',
- 'bp_draft': 'New Blueprints',
- 'bp_implementation': 'Completed Blueprints',
+ 'bpd': 'New Blueprints',
+ 'bpc': 'Completed Blueprints',
}
METRIC_TO_RECORD_TYPE = {
@@ -58,8 +58,8 @@ METRIC_TO_RECORD_TYPE = {
'commits': 'commit',
'marks': 'mark',
'emails': 'email',
- 'bp_draft': 'bp_draft',
- 'bp_implementation': 'bp_implementation',
+ 'bpd': 'bpd',
+ 'bpc': 'bpc',
}
DEFAULT_RECORDS_LIMIT = 10
@@ -406,8 +406,8 @@ def aggregate_filter():
'loc': (loc_filter, None),
'marks': (mark_filter, mark_finalize),
'emails': (incremental_filter, None),
- 'bp_draft': (incremental_filter, None),
- 'bp_implementation': (incremental_filter, None),
+ 'bpd': (incremental_filter, None),
+ 'bpc': (incremental_filter, None),
}
if metric not in metric_to_filters_map:
raise Exception('Invalid metric %s' % metric)
@@ -554,44 +554,6 @@ def page_not_found(e):
pass
-def contribution_details(records):
- blueprints_map = {}
- bugs_map = {}
- marks = dict((m, 0) for m in [-2, -1, 0, 1, 2])
- commit_count = 0
- loc = 0
-
- for record in records:
- if 'blueprint_id' in record:
- for bp in record['blueprint_id']:
- blueprints_map[bp] = record
- if 'bug_id' in record:
- for bug in record['bug_id']:
- bugs_map[bug] = record
-
- if record['record_type'] == 'mark':
- marks[int(record['value'])] += 1
- elif record['record_type'] == 'commit':
- commit_count += 1
- loc += record['loc']
-
- blueprints = sorted([{'id': key, 'module': value['module']}
- for key, value in blueprints_map.iteritems()],
- key=lambda x: x['id'])
- bugs = sorted([{'id': key, 'module': value['module']}
- for key, value in bugs_map.iteritems()],
- key=lambda x: int(x['id']))
-
- result = {
- 'blueprints': blueprints,
- 'bugs': bugs,
- 'commit_count': commit_count,
- 'loc': loc,
- 'marks': marks,
- }
- return result
-
-
# AJAX Handlers ---------
def _get_aggregated_stats(records, metric_filter, keys, param_id,
@@ -653,7 +615,9 @@ def _extend_record(record):
record['company_link'] = make_link(
record['company_name'], '/',
{'company': record['company_name'], 'user_id': ''})
- record['gravatar'] = gravatar(record['author_email'])
+ record['gravatar'] = gravatar(record.get('author_email', 'stackalytics'))
+ record['blueprint_id_count'] = len(record.get('blueprint_id', []))
+ record['bug_id_count'] = len(record.get('bug_id', []))
@app.route('/api/1.0/activity')
@@ -693,10 +657,13 @@ def get_activity_json(records):
_extend_record(email)
email['email_link'] = email.get('email_link') or ''
result.append(email)
- elif ((record['record_type'] == 'bp_draft') or
- (record['record_type'] == 'bp_implementation')):
+ elif ((record['record_type'] == 'bpd') or
+ (record['record_type'] == 'bpc')):
blueprint = record.copy()
_extend_record(blueprint)
+ if 'mention_date' in record:
+ record['mention_date_str'] = format_datetime(
+ record['mention_date'])
result.append(blueprint)
result.sort(key=lambda x: x['date'], reverse=True)
@@ -708,7 +675,36 @@ def get_activity_json(records):
@exception_handler()
@record_filter(ignore='metric')
def get_contribution_json(records):
- return contribution_details(records)
+ marks = dict((m, 0) for m in [-2, -1, 0, 1, 2])
+ commit_count = 0
+ loc = 0
+ new_blueprint_count = 0
+ competed_blueprint_count = 0
+ email_count = 0
+
+ for record in records:
+ record_type = record['record_type']
+ if record_type == 'commit':
+ commit_count += 1
+ loc += record['loc']
+ elif record['record_type'] == 'mark':
+ marks[int(record['value'])] += 1
+ elif record['record_type'] == 'email':
+ email_count += 1
+ elif record['record_type'] == 'bpd':
+ new_blueprint_count += 1
+ elif record['record_type'] == 'bpc':
+ competed_blueprint_count += 1
+
+ result = {
+ 'new_blueprint_count': new_blueprint_count,
+ 'competed_blueprint_count': competed_blueprint_count,
+ 'commit_count': commit_count,
+ 'email_count': email_count,
+ 'loc': loc,
+ 'marks': marks,
+ }
+ return result
@app.route('/api/1.0/companies')
@@ -817,7 +813,10 @@ def get_user(user_id):
company_name, '/', {'company': company_name, 'user_id': ''})
else:
user['company_link'] = ''
- user['gravatar'] = gravatar(user['emails'][0])
+ if user['emails']:
+ user['gravatar'] = gravatar(user['emails'][0])
+ else:
+ user['gravatar'] = gravatar('stackalytics')
return user
diff --git a/etc/test_default_data.json b/etc/test_default_data.json
index f364ed23b..5372bf2fe 100644
--- a/etc/test_default_data.json
+++ b/etc/test_default_data.json
@@ -119,7 +119,7 @@
"releases": [
{
"release_name": "prehistory",
- "end_date": "2011-Apr-21"
+ "end_date": "2011-Apr-22"
},
{
"release_name": "Diablo",
diff --git a/stackalytics/processor/launchpad_utils.py b/stackalytics/processor/launchpad_utils.py
new file mode 100644
index 000000000..2028ae9a9
--- /dev/null
+++ b/stackalytics/processor/launchpad_utils.py
@@ -0,0 +1,66 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import httplib
+import urlparse
+
+from stackalytics.openstack.common import log as logging
+from stackalytics.processor import utils
+
+LOG = logging.getLogger(__name__)
+
+
+LP_URI_V1 = 'https://api.launchpad.net/1.0/%s'
+LP_URI_DEVEL = 'https://api.launchpad.net/devel/%s'
+
+
+def lp_profile_by_launchpad_id(launchpad_id):
+ LOG.debug('Lookup user id %s at Launchpad', launchpad_id)
+ uri = LP_URI_V1 % ('~' + launchpad_id)
+ return utils.read_json_from_uri(uri)
+
+
+def lp_profile_by_email(email):
+ LOG.debug('Lookup user email %s at Launchpad', email)
+ uri = LP_URI_V1 % ('people/?ws.op=getByEmail&email=' + email)
+ return utils.read_json_from_uri(uri)
+
+
+def lp_module_exists(module):
+ uri = LP_URI_DEVEL % module
+ parsed_uri = urlparse.urlparse(uri)
+ conn = httplib.HTTPConnection(parsed_uri.netloc)
+ conn.request('GET', parsed_uri.path)
+ res = conn.getresponse()
+ LOG.debug('Checked uri: %(uri)s, status: %(status)s',
+ {'uri': uri, 'status': res.status})
+ conn.close()
+ return res.status != 404
+
+
+def lp_blueprint_generator(module):
+ uri = LP_URI_DEVEL % (module + '/all_specifications')
+ while uri:
+ LOG.debug('Reading chunk from uri %s', uri)
+ chunk = utils.read_json_from_uri(uri)
+
+ if not chunk:
+ LOG.warn('No data was read from uri %s', uri)
+ break
+
+ for record in chunk['entries']:
+ yield record
+
+ uri = chunk.get('next_collection_link')
diff --git a/stackalytics/processor/lp.py b/stackalytics/processor/lp.py
index 353ceca6e..94a9c919c 100644
--- a/stackalytics/processor/lp.py
+++ b/stackalytics/processor/lp.py
@@ -13,10 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import httplib
-import urlparse
-
from stackalytics.openstack.common import log as logging
+from stackalytics.processor import launchpad_utils
from stackalytics.processor import utils
@@ -27,18 +25,6 @@ LINK_FIELDS = ['owner', 'drafter', 'starter', 'completer',
DATE_FIELDS = ['date_created', 'date_completed', 'date_started']
-def _module_exists(module):
- uri = 'https://api.launchpad.net/devel/%s' % module
- parsed_uri = urlparse.urlparse(uri)
- conn = httplib.HTTPConnection(parsed_uri.netloc)
- conn.request('GET', parsed_uri.path)
- res = conn.getresponse()
- LOG.debug('Checked uri: %(uri)s, status: %(status)s',
- {'uri': uri, 'status': res.status})
- conn.close()
- return res.status != 404
-
-
def _link_to_launchpad_id(link):
return link[link.find('~') + 1:]
@@ -47,30 +33,22 @@ def log(repo):
module = repo['module']
LOG.debug('Retrieving list of blueprints for module: %s', module)
- if not _module_exists(module):
+ if not launchpad_utils.lp_module_exists(module):
LOG.debug('Module %s not exist at Launchpad', module)
return
- uri = 'https://api.launchpad.net/devel/%s/all_specifications' % module
- while True:
- LOG.debug('Reading chunk from uri %s', uri)
- chunk = utils.read_json_from_uri(uri)
- if 'next_collection_link' not in chunk:
- break
- uri = chunk['next_collection_link']
+ for record in launchpad_utils.lp_blueprint_generator(module):
+ for field in LINK_FIELDS:
+ link = record[field + '_link']
+ if link:
+ record[field] = _link_to_launchpad_id(link)
+ del record[field + '_link']
+ for field in DATE_FIELDS:
+ date = record[field]
+ if date:
+ record[field] = utils.iso8601_to_timestamp(date)
- for record in chunk['entries']:
- for field in LINK_FIELDS:
- link = record[field + '_link']
- if link:
- record[field] = _link_to_launchpad_id(link)
- del record[field + '_link']
- for field in DATE_FIELDS:
- date = record[field]
- if date:
- record[field] = utils.iso8601_to_timestamp(date)
+ record['module'] = module
- record['module'] = module
-
- LOG.debug('New blueprint: %s', record)
- yield record
+ LOG.debug('New blueprint: %s', record)
+ yield record
diff --git a/stackalytics/processor/normalizer.py b/stackalytics/processor/normalizer.py
index 8497eba9b..4f033fcbd 100644
--- a/stackalytics/processor/normalizer.py
+++ b/stackalytics/processor/normalizer.py
@@ -44,7 +44,10 @@ def normalize_user(user):
return cmp(x["end_date"], y["end_date"])
user['companies'].sort(cmp=end_date_comparator)
- user['user_id'] = get_user_id(user['launchpad_id'], user['emails'][0])
+ if user['emails']:
+ user['user_id'] = get_user_id(user['launchpad_id'], user['emails'][0])
+ else:
+ user['user_id'] = user['launchpad_id']
def _normalize_users(users):
diff --git a/stackalytics/processor/record_processor.py b/stackalytics/processor/record_processor.py
index b1eb83a77..430987155 100644
--- a/stackalytics/processor/record_processor.py
+++ b/stackalytics/processor/record_processor.py
@@ -16,6 +16,7 @@
import bisect
from stackalytics.openstack.common import log as logging
+from stackalytics.processor import launchpad_utils
from stackalytics.processor import normalizer
from stackalytics.processor import utils
@@ -66,6 +67,9 @@ class RecordProcessor(object):
return companies[-1]['company_name']
def _get_company_by_email(self, email):
+ if not email:
+ return None
+
name, at, domain = email.partition('@')
if domain:
parts = domain.split('.')
@@ -81,13 +85,16 @@ class RecordProcessor(object):
user = {
'user_id': normalizer.get_user_id(launchpad_id, email),
'launchpad_id': launchpad_id,
- 'user_name': user_name,
- 'emails': [email],
+ 'user_name': user_name or '',
'companies': [{
'company_name': company,
'end_date': 0,
}],
}
+ if email:
+ user['emails'] = [email]
+ else:
+ user['emails'] = []
normalizer.normalize_user(user)
LOG.debug('Create new user: %s', user)
return user
@@ -95,12 +102,9 @@ class RecordProcessor(object):
def _get_lp_info(self, email):
lp_profile = None
if not utils.check_email_validity(email):
- LOG.debug('User email is not valid %s' % email)
+ LOG.debug('User email is not valid %s', email)
else:
- LOG.debug('Lookup user email %s at Launchpad' % email)
- uri = ('https://api.launchpad.net/1.0/people/?'
- 'ws.op=getByEmail&email=%s' % email)
- lp_profile = utils.read_json_from_uri(uri)
+ lp_profile = launchpad_utils.lp_profile_by_email(email)
if not lp_profile:
LOG.debug('User with email %s not found', email)
@@ -109,6 +113,18 @@ class RecordProcessor(object):
LOG.debug('Email is mapped to launchpad user: %s', lp_profile['name'])
return lp_profile['name'], lp_profile['display_name']
+ def _get_lp_user_name(self, launchpad_id):
+ if not launchpad_id:
+ return None
+
+ lp_profile = launchpad_utils.lp_profile_by_launchpad_id(launchpad_id)
+
+ if not lp_profile:
+ LOG.debug('User with id %s not found', launchpad_id)
+ return launchpad_id
+
+ return lp_profile['display_name']
+
def _get_independent(self):
return self.domains_index['']
@@ -124,14 +140,14 @@ class RecordProcessor(object):
self.updated_users.add(user['user_id'])
def update_user(self, record):
- email = record['author_email']
+ email = record.get('author_email')
if email in self.users_index:
user = self.users_index[email]
else:
- if ('launchpad_id' in record) and (record['launchpad_id']):
- launchpad_id = record['launchpad_id']
- user_name = record['author_name']
+ if record.get('launchpad_id'):
+ launchpad_id = record.get('launchpad_id')
+ user_name = record.get('author_name')
else:
launchpad_id, user_name = self._get_lp_info(email)
@@ -142,11 +158,14 @@ class RecordProcessor(object):
else:
# create new
if not user_name:
- user_name = record['author_name']
+ user_name = record.get('author_name')
+ if not user_name:
+ user_name = self._get_lp_user_name(launchpad_id)
user = self._create_user(launchpad_id, email, user_name)
utils.store_user(self.runtime_storage_inst, user)
- self.users_index[email] = user
+ if email:
+ self.users_index[email] = user
if user['launchpad_id']:
self.users_index[user['launchpad_id']] = user
@@ -158,12 +177,12 @@ class RecordProcessor(object):
record['user_id'] = user['user_id']
record['launchpad_id'] = user['launchpad_id']
- if ('user_name' in user) and (user['user_name']):
+ if user.get('user_name'):
record['author_name'] = user['user_name']
company = self._find_company(user['companies'], record['date'])
if company != '*robots':
- company = (self._get_company_by_email(record['author_email'])
+ company = (self._get_company_by_email(record.get('author_email'))
or company)
record['company_name'] = company
@@ -199,7 +218,7 @@ class RecordProcessor(object):
review_id = record['id']
module = record['module']
- for patch in record['patchSets']:
+ for patch in record.get('patchSets', []):
if 'approvals' not in patch:
continue # not reviewed by anyone
for approval in patch['approvals']:
@@ -265,40 +284,28 @@ class RecordProcessor(object):
yield record
def _process_blueprint(self, record):
- if record.get('drafter'):
- bp_draft = dict([(k, v) for k, v in record.iteritems()])
- bp_draft['primary_key'] = 'bpd:' + record['self_link']
+ bpd_author = record.get('drafter') or record.get('owner')
- drafter = utils.load_user(self.runtime_storage_inst,
- record['drafter'])
- if drafter and record['date_created']:
- bp_draft['record_type'] = 'bp_draft'
- bp_draft['author_name'] = drafter['user_name']
- bp_draft['author_email'] = drafter['emails'][0]
- bp_draft['launchpad_id'] = record['drafter']
- bp_draft['date'] = record['date_created']
+ bpd = dict([(k, v) for k, v in record.iteritems()])
+ bpd['record_type'] = 'bpd'
+ bpd['primary_key'] = 'bpd:' + record['self_link']
+ bpd['launchpad_id'] = bpd_author
+ bpd['date'] = record['date_created']
- self._update_record_and_user(bp_draft)
+ self._update_record_and_user(bpd)
- yield bp_draft
+ yield bpd
- if record.get('assignee'):
- bp_implementation = dict([(k, v) for k, v in record.iteritems()])
- bp_implementation['primary_key'] = 'bpi:' + record['self_link']
+ if record.get('assignee') and record['date_completed']:
+ bpc = dict([(k, v) for k, v in record.iteritems()])
+ bpc['record_type'] = 'bpc'
+ bpc['primary_key'] = 'bpc:' + record['self_link']
+ bpc['launchpad_id'] = record['assignee']
+ bpc['date'] = record['date_completed']
- assignee = utils.load_user(self.runtime_storage_inst,
- record['assignee'])
- if assignee and record['date_completed']:
- bp_implementation['record_type'] = 'bp_implementation'
- bp_implementation['author_name'] = assignee['user_name']
- bp_implementation['author_email'] = assignee['emails'][0]
- bp_implementation['launchpad_id'] = record['assignee']
- bp_implementation['date'] = record['date_completed']
+ self._update_record_and_user(bpc)
- if bp_implementation['author_email']:
- self._update_record_and_user(bp_implementation)
-
- yield bp_implementation
+ yield bpc
def _apply_type_based_processing(self, record):
if record['record_type'] == 'commit':
@@ -359,16 +366,69 @@ class RecordProcessor(object):
self.runtime_storage_inst.set_by_key('users', self.users_index)
def _get_records_for_users_to_update(self):
+ valid_blueprints = {}
+ mentioned_blueprints = {}
for record in self.runtime_storage_inst.get_all_records():
+ for bp in record.get('blueprint_id', []):
+ if bp in mentioned_blueprints:
+ mentioned_blueprints[bp]['count'] += 1
+ if record['date'] > mentioned_blueprints[bp]['date']:
+ mentioned_blueprints[bp]['date'] = record['date']
+ else:
+ mentioned_blueprints[bp] = {
+ 'count': 1,
+ 'date': record['date']
+ }
+ if record['record_type'] in ['bpd', 'bpi']:
+ valid_blueprints[record['name']] = {
+ 'primary_key': record['primary_key'],
+ 'count': 0,
+ 'date': record['date']
+ }
+
+ for bp in valid_blueprints.keys():
+ if bp in mentioned_blueprints:
+ valid_blueprints[bp]['count'] = (
+ mentioned_blueprints[bp]['count'])
+ valid_blueprints[bp]['date'] = (
+ mentioned_blueprints[bp]['date'])
+
+ for record in self.runtime_storage_inst.get_all_records():
+
+ need_update = False
+
user_id = record['user_id']
if user_id in self.updated_users:
user = self.users_index[user_id]
user_company_name = user['companies'][0]['company_name']
if record['company_name'] != user_company_name:
- LOG.debug('Record company will be changed to: %s',
- user_company_name)
+ LOG.debug('Update record %s: company changed to: %s',
+ record['primary_key'], user_company_name)
record['company_name'] = user_company_name
- yield record
+ need_update = True
+
+ valid_bp = set([])
+ for bp in record.get('blueprint_id', []):
+ if bp in valid_blueprints:
+ valid_bp.add(bp)
+ else:
+ LOG.debug('Update record %s: removed invalid bp: %s',
+ record['primary_key'], bp)
+ need_update = True
+ record['blueprint_id'] = list(valid_bp)
+
+ if record['record_type'] in ['bpd', 'bpi']:
+ bp = valid_blueprints[record['name']]
+ if ((record.get('mention_count') != bp['count']) or
+ (record.get('mention_date') != bp['date'])):
+ record['mention_count'] = bp['count']
+ record['mention_date'] = bp['date']
+ LOG.debug('Update record %s: mention stats: (%s:%s)',
+ record['primary_key'], bp['count'], bp['date'])
+ need_update = True
+
+ if need_update:
+ yield record
def finalize(self):
self.runtime_storage_inst.set_records(
diff --git a/stackalytics/processor/utils.py b/stackalytics/processor/utils.py
index 452c41f0a..abb1b1648 100644
--- a/stackalytics/processor/utils.py
+++ b/stackalytics/processor/utils.py
@@ -63,7 +63,10 @@ def read_uri(uri):
def read_json_from_uri(uri):
- return json.loads(read_uri(uri))
+ try:
+ return json.loads(read_uri(uri))
+ except Exception as e:
+ LOG.warn('Error parsing json: %s' % e)
def make_range(start, stop, step):
diff --git a/tests/unit/test_record_processor.py b/tests/unit/test_record_processor.py
index d3a429cae..02d710c6f 100644
--- a/tests/unit/test_record_processor.py
+++ b/tests/unit/test_record_processor.py
@@ -21,38 +21,6 @@ from stackalytics.processor import runtime_storage
from stackalytics.processor import utils
-LP_URI = 'https://api.launchpad.net/1.0/people/?ws.op=getByEmail&email=%s'
-
-COMPANIES = [
- {
- 'company_name': 'SuperCompany',
- 'domains': ['super.com', 'super.no']
- },
- {
- "domains": ["nec.com", "nec.co.jp"],
- "company_name": "NEC"
- },
- {
- 'company_name': '*independent',
- 'domains': ['']
- },
-]
-
-USERS = [
- {
- 'user_id': 'john_doe',
- 'launchpad_id': 'john_doe',
- 'user_name': 'John Doe',
- 'emails': ['johndoe@gmail.com', 'jdoe@super.no'],
- 'companies': [
- {'company_name': '*independent',
- 'end_date': 1234567890},
- {'company_name': 'SuperCompany',
- 'end_date': 0},
- ]
- }
-]
-
RELEASES = [
{
'release_name': 'prehistory',
@@ -84,164 +52,423 @@ class TestRecordProcessor(testtools.TestCase):
self.read_json_from_uri_patch = mock.patch(
'stackalytics.processor.utils.read_json_from_uri')
self.read_launchpad = self.read_json_from_uri_patch.start()
+ self.lp_profile_by_launchpad_id_patch = mock.patch(
+ 'stackalytics.processor.launchpad_utils.'
+ 'lp_profile_by_launchpad_id')
+ self.lp_profile_by_launchpad_id = (
+ self.lp_profile_by_launchpad_id_patch.start())
+ self.lp_profile_by_launchpad_id.return_value = None
+ self.lp_profile_by_email_patch = mock.patch(
+ 'stackalytics.processor.launchpad_utils.lp_profile_by_email')
+ self.lp_profile_by_email = (
+ self.lp_profile_by_email_patch.start())
+ self.lp_profile_by_email.return_value = None
def tearDown(self):
super(TestRecordProcessor, self).tearDown()
self.read_json_from_uri_patch.stop()
+ self.lp_profile_by_launchpad_id_patch.stop()
+ self.lp_profile_by_email_patch.stop()
+
+ # get_company_by_email
def test_get_company_by_email_mapped(self):
- record_processor_inst = make_record_processor()
- email = 'jdoe@super.no'
+ record_processor_inst = self.make_record_processor(
+ companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}]
+ )
+ email = 'jdoe@ibm.com'
res = record_processor_inst._get_company_by_email(email)
- self.assertEquals('SuperCompany', res)
+ self.assertEquals('IBM', res)
def test_get_company_by_email_with_long_suffix_mapped(self):
- record_processor_inst = make_record_processor()
+ record_processor_inst = self.make_record_processor(
+ companies=[{'company_name': 'NEC', 'domains': ['nec.co.jp']}]
+ )
email = 'man@mxw.nes.nec.co.jp'
res = record_processor_inst._get_company_by_email(email)
self.assertEquals('NEC', res)
def test_get_company_by_email_with_long_suffix_mapped_2(self):
- record_processor_inst = make_record_processor()
+ record_processor_inst = self.make_record_processor(
+ companies=[{'company_name': 'NEC',
+ 'domains': ['nec.co.jp', 'nec.com']}]
+ )
email = 'man@mxw.nes.nec.com'
res = record_processor_inst._get_company_by_email(email)
self.assertEquals('NEC', res)
def test_get_company_by_email_not_mapped(self):
- record_processor_inst = make_record_processor()
+ record_processor_inst = self.make_record_processor()
email = 'foo@boo.com'
res = record_processor_inst._get_company_by_email(email)
self.assertEquals(None, res)
- def test_update_commit_existing_user(self):
- record_processor_inst = make_record_processor()
- commit_generator = generate_commits()
- commit = list(record_processor_inst.process(commit_generator))[0]
+ # get_lp_info
- self.assertEquals('SuperCompany', commit['company_name'])
- self.assertEquals('john_doe', commit['launchpad_id'])
+ def test_get_lp_info_invalid_email(self):
+ self.read_launchpad.return_value = None
+ record_processor_inst = self.make_record_processor(users=[])
+ self.assertEquals((None, None),
+ record_processor_inst._get_lp_info('error.root'))
- def test_update_commit_existing_user_old_job(self):
- record_processor_inst = make_record_processor()
- commit_generator = generate_commits(date=1000000000)
- commit = list(record_processor_inst.process(commit_generator))[0]
+ # commit processing
- self.assertEquals('*independent', commit['company_name'])
- self.assertEquals('john_doe', commit['launchpad_id'])
+ def test_process_commit_existing_user(self):
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+ 'companies': [
+ {'company_name': '*independent',
+ 'end_date': 1234567890},
+ {'company_name': 'NEC',
+ 'end_date': 0},
+ ]
+ }
+ ])
- def test_update_commit_existing_user_new_email_known_company(self):
+ processed_commit = list(record_processor_inst.process(
+ generate_commits(author_email='johndoe@gmail.com',
+ author_name='John Doe')))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@gmail.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'NEC',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
+
+ def test_process_commit_existing_user_old_job(self):
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+ 'companies': [
+ {'company_name': '*independent',
+ 'end_date': 1234567890},
+ {'company_name': 'NEC',
+ 'end_date': 0},
+ ]
+ }
+ ])
+
+ processed_commit = list(record_processor_inst.process(
+ generate_commits(author_email='johndoe@gmail.com',
+ author_name='John Doe',
+ date=1000000000)))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@gmail.com',
+ 'author_name': 'John Doe',
+ 'company_name': '*independent',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
+
+ def test_process_commit_existing_user_new_email_known_company(self):
# User is known to LP, his email is new to us, and maps to other
# company. Should return other company instead of those mentioned
- # in user db
- email = 'johndoe@nec.co.jp'
- commit_generator = generate_commits(email=email)
- launchpad_id = 'john_doe'
- self.read_launchpad.return_value = {'name': launchpad_id,
- 'display_name': launchpad_id}
- user = make_user()
- record_processor_inst = make_record_processor(
- make_runtime_storage(users=[user]))
+ # in user profile
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@nec.co.jp'],
+ 'companies': [{'company_name': 'NEC', 'end_date': 0}]}
+ ],
+ companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
+ lp_info={'johndoe@ibm.com':
+ {'name': 'john_doe', 'display_name': 'John Doe'}})
- commit = list(record_processor_inst.process(commit_generator))[0]
+ processed_commit = list(record_processor_inst.process(
+ generate_commits(author_email='johndoe@ibm.com',
+ author_name='John Doe')))[0]
- self.read_launchpad.assert_called_once_with(LP_URI % email)
- self.assertIn(email, user['emails'])
- self.assertEquals('NEC', commit['company_name'])
- self.assertEquals(launchpad_id, commit['launchpad_id'])
-
- def test_update_commit_existing_user_new_email_unknown_company(self):
- # User is known to LP, but his email is new to us. Should match
- # the user and return current company
- email = 'johndoe@yahoo.com'
- commit_generator = generate_commits(email=email)
- launchpad_id = 'john_doe'
- self.read_launchpad.return_value = {'name': launchpad_id,
- 'display_name': launchpad_id}
- user = make_user()
- record_processor_inst = make_record_processor(
- make_runtime_storage(users=[user]))
-
- commit = list(record_processor_inst.process(commit_generator))[0]
-
- self.read_launchpad.assert_called_once_with(LP_URI % email)
- self.assertIn(email, user['emails'])
- self.assertEquals('SuperCompany', commit['company_name'])
- self.assertEquals(launchpad_id, commit['launchpad_id'])
-
- def test_update_commit_existing_user_new_email_known_company_update(self):
- # User is known to LP, his email is new to us and belongs to company B.
- # Should match the user and return company B and update user
- email = 'johndoe@nec.com'
- commit_generator = generate_commits(email=email)
- launchpad_id = 'john_doe'
- self.read_launchpad.return_value = {'name': launchpad_id,
- 'display_name': launchpad_id}
- user = {
- 'user_id': 'john_doe',
- 'launchpad_id': launchpad_id,
- 'user_name': 'John Doe',
- 'emails': ['johndoe@gmail.com'],
- 'companies': [
- {'company_name': '*independent', 'end_date': 0}
- ]
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@ibm.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'IBM',
}
- record_processor_inst = make_record_processor(
- make_runtime_storage(users=[user]))
- commit = list(record_processor_inst.process(commit_generator))[0]
+ self.assertRecordsMatch(expected_commit, processed_commit)
+ self.assertIn('johndoe@ibm.com',
+ record_processor_inst.users_index['john_doe']['emails'])
- self.read_launchpad.assert_called_once_with(LP_URI % email)
- self.assertIn(email, user['emails'])
- self.assertEquals('NEC', user['companies'][0]['company_name'],
+ def test_process_commit_existing_user_new_email_unknown_company(self):
+ # User is known to LP, but his email is new to us. Should match
+ # the user and return company from user profile
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@nec.co.jp'],
+ 'companies': [{'company_name': 'NEC', 'end_date': 0}]}
+ ],
+ companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
+ lp_info={'johndoe@gmail.com':
+ {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+ processed_commit = list(record_processor_inst.process(
+ generate_commits(author_email='johndoe@gmail.com',
+ author_name='John Doe')))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@gmail.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'NEC',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
+ self.assertIn('johndoe@gmail.com',
+ record_processor_inst.users_index['john_doe']['emails'])
+
+ def test_process_commit_existing_user_new_email_known_company_update(self):
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@gmail.com'],
+ 'companies': [{'company_name': '*independent',
+ 'end_date': 0}]}
+ ],
+ companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
+ lp_info={'johndoe@ibm.com':
+ {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+ processed_commit = list(record_processor_inst.process(
+ generate_commits(author_email='johndoe@ibm.com',
+ author_name='John Doe')))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@ibm.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'IBM',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
+ user = record_processor_inst.users_index['john_doe']
+ self.assertIn('johndoe@gmail.com', user['emails'])
+ self.assertEquals('IBM', user['companies'][0]['company_name'],
message='User affiliation should be updated')
- self.assertEquals('NEC', commit['company_name'])
- self.assertEquals(launchpad_id, commit['launchpad_id'])
- def test_update_commit_new_user(self):
+ def test_process_commit_new_user(self):
# User is known to LP, but new to us
# Should add new user and set company depending on email
- email = 'smith@nec.com'
- commit_generator = generate_commits(email=email)
- launchpad_id = 'smith'
- self.read_launchpad.return_value = {'name': launchpad_id,
- 'display_name': 'Smith'}
- record_processor_inst = make_record_processor(
- make_runtime_storage(users=[]))
+ record_processor_inst = self.make_record_processor(
+ companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
+ lp_info={'johndoe@ibm.com':
+ {'name': 'john_doe', 'display_name': 'John Doe'}})
- commit = list(record_processor_inst.process(commit_generator))[0]
+ processed_commit = list(record_processor_inst.process(
+ generate_commits(author_email='johndoe@ibm.com',
+ author_name='John Doe')))[0]
- self.read_launchpad.assert_called_once_with(LP_URI % email)
- self.assertEquals('NEC', commit['company_name'])
- self.assertEquals(launchpad_id, commit['launchpad_id'])
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@ibm.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'IBM',
+ }
- def test_update_commit_new_user_unknown_to_lb(self):
+ self.assertRecordsMatch(expected_commit, processed_commit)
+ user = record_processor_inst.users_index['john_doe']
+ self.assertIn('johndoe@ibm.com', user['emails'])
+ self.assertEquals('IBM', user['companies'][0]['company_name'])
+
+ def test_process_commit_new_user_unknown_to_lb(self):
# User is new to us and not known to LP
# Should set user name and empty LPid
- email = 'inkognito@avs.com'
- commit_generator = generate_commits(email=email)
- self.read_launchpad.return_value = None
- record_processor_inst = make_record_processor(
- make_runtime_storage(users=[]))
+ record_processor_inst = self.make_record_processor(
+ companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}])
- commit = list(record_processor_inst.process(commit_generator))[0]
+ processed_commit = list(record_processor_inst.process(
+ generate_commits(author_email='johndoe@ibm.com',
+ author_name='John Doe')))[0]
- self.read_launchpad.assert_called_once_with(LP_URI % email)
- self.assertEquals('*independent', commit['company_name'])
- self.assertEquals(None, commit['launchpad_id'])
+ expected_commit = {
+ 'launchpad_id': None,
+ 'author_email': 'johndoe@ibm.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'IBM',
+ }
- def test_update_commit_invalid_email(self):
- # User's email is malformed
- email = 'error.root'
- commit_generator = generate_commits(email=email)
- self.read_launchpad.return_value = None
- record_processor_inst = make_record_processor(
- make_runtime_storage(users=[]))
+ self.assertRecordsMatch(expected_commit, processed_commit)
+ self.assertEquals(1, len(record_processor_inst.users_index))
+ user = record_processor_inst.users_index['johndoe@ibm.com']
+ self.assertIn('johndoe@ibm.com', user['emails'])
+ self.assertEquals('IBM', user['companies'][0]['company_name'])
+ self.assertEquals(None, user['launchpad_id'])
- commit = list(record_processor_inst.process(commit_generator))[0]
+ # process records complex scenarios
- self.assertEquals(0, self.read_launchpad.called)
- self.assertEquals('*independent', commit['company_name'])
- self.assertEquals(None, commit['launchpad_id'])
+ def test_process_blueprint_one_draft_spawned_lp_doesnt_know_user(self):
+ # In: blueprint record
+ # LP doesn't know user
+ # Out: blueprint-draft record
+ # new user profile created
+ record_processor_inst = self.make_record_processor()
+
+ processed_records = list(record_processor_inst.process([
+ {'record_type': 'bp',
+ 'self_link': 'http://launchpad.net/blueprint',
+ 'owner': 'john_doe',
+ 'date_created': 1234567890}
+ ]))
+
+ self.assertRecordsMatch(
+ {'record_type': 'bpd',
+ 'launchpad_id': 'john_doe',
+ 'author_name': 'john_doe',
+ 'company_name': '*independent'},
+ processed_records[0])
+
+ self.assertEquals(1, len(record_processor_inst.users_index))
+ user = record_processor_inst.users_index['john_doe']
+ self.assertEquals({
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'john_doe',
+ 'emails': [],
+ 'companies': [{'company_name': '*independent', 'end_date': 0}]
+ }, user)
+
+ def test_process_blueprint_one_draft_spawned_lp_knows_user(self):
+ # In: blueprint record
+ # LP knows user
+ # Out: blueprint-draft record
+ # new user profile created, name is taken from LP profile
+ record_processor_inst = self.make_record_processor(
+ lp_user_name={
+ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+ processed_records = list(record_processor_inst.process([
+ {'record_type': 'bp',
+ 'self_link': 'http://launchpad.net/blueprint',
+ 'owner': 'john_doe',
+ 'date_created': 1234567890}
+ ]))
+
+ self.assertRecordsMatch(
+ {'record_type': 'bpd',
+ 'launchpad_id': 'john_doe',
+ 'author_name': 'John Doe',
+ 'company_name': '*independent'},
+ processed_records[0])
+
+ self.assertEquals(1, len(record_processor_inst.users_index))
+ user = record_processor_inst.users_index['john_doe']
+ self.assertEquals({
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': [],
+ 'companies': [{'company_name': '*independent', 'end_date': 0}]
+ }, user)
+
+ def test_process_blueprint_then_review(self):
+ record_processor_inst = self.make_record_processor(
+ lp_user_name={
+ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+ processed_records = list(record_processor_inst.process([
+ {'record_type': 'bp',
+ 'self_link': 'http://launchpad.net/blueprint',
+ 'owner': 'john_doe',
+ 'date_created': 1234567890},
+ {'record_type': 'review',
+ 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
+ 'subject': 'Fix AttributeError in Keypair._add_details()',
+ 'owner': {'name': 'John Doe',
+ 'email': 'john_doe@gmail.com',
+ 'username': 'john_doe'},
+ 'createdOn': 1379404951,
+ 'module': 'nova'}
+ ]))
+
+ self.assertRecordsMatch(
+ {'record_type': 'bpd',
+ 'launchpad_id': 'john_doe',
+ 'author_name': 'John Doe',
+ 'company_name': '*independent'},
+ processed_records[0])
+
+ self.assertRecordsMatch(
+ {'record_type': 'review',
+ 'launchpad_id': 'john_doe',
+ 'author_name': 'John Doe',
+ 'author_email': 'john_doe@gmail.com',
+ 'company_name': '*independent'},
+ processed_records[1])
+
+ user = {'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['john_doe@gmail.com'],
+ 'companies': [{'company_name': '*independent', 'end_date': 0}]}
+ self.assertEquals({'john_doe': user, 'john_doe@gmail.com': user},
+ record_processor_inst.users_index)
+
+ def test_process_blueprint_then_commit(self):
+ record_processor_inst = self.make_record_processor(
+ lp_user_name={
+ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}},
+ lp_info={'john_doe@gmail.com':
+ {'name': 'john_doe', 'display_name': 'John Doe'}})
+
+ processed_records = list(record_processor_inst.process([
+ {'record_type': 'bp',
+ 'self_link': 'http://launchpad.net/blueprint',
+ 'owner': 'john_doe',
+ 'date_created': 1234567890},
+ {'record_type': 'commit',
+ 'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
+ 'author_name': 'John Doe',
+ 'author_email': 'john_doe@gmail.com',
+ 'date': 1234567890,
+ 'lines_added': 25,
+ 'lines_deleted': 9,
+ 'release_name': 'havana'}
+ ]))
+
+ self.assertRecordsMatch(
+ {'record_type': 'bpd',
+ 'launchpad_id': 'john_doe',
+ 'author_name': 'John Doe',
+ 'company_name': '*independent'},
+ processed_records[0])
+
+ self.assertRecordsMatch(
+ {'record_type': 'commit',
+ 'launchpad_id': 'john_doe',
+ 'author_name': 'John Doe',
+ 'author_email': 'john_doe@gmail.com',
+ 'company_name': '*independent'},
+ processed_records[1])
+
+ user = {'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['john_doe@gmail.com'],
+ 'companies': [{'company_name': '*independent', 'end_date': 0}]}
+ self.assertEquals({'john_doe': user, 'john_doe@gmail.com': user},
+ record_processor_inst.users_index)
+
+ # update records
def _generate_record_commit(self):
yield {'commit_id': u'0afdc64bfd041b03943ceda7849c4443940b6053',
@@ -272,68 +499,187 @@ class TestRecordProcessor(testtools.TestCase):
def test_update_record_no_changes(self):
commit_generator = self._generate_record_commit()
release_index = {'0afdc64bfd041b03943ceda7849c4443940b6053': 'havana'}
- record_processor_inst = make_record_processor(
- make_runtime_storage(users=[]))
+ record_processor_inst = self.make_record_processor(
+ users=[],
+ companies=[{'company_name': 'SuperCompany',
+ 'domains': ['super.no']}])
updated = list(record_processor_inst.update(commit_generator,
release_index))
self.assertEquals(0, len(updated))
- def test_process_mail(self):
- record_processor_inst = make_record_processor()
- commit_generator = generate_emails(
- subject='[openstack-dev] [Stackalytics] Configuration files')
- commit = list(record_processor_inst.process(commit_generator))[0]
+ # mail processing
- self.assertEquals('SuperCompany', commit['company_name'])
- self.assertEquals('john_doe', commit['launchpad_id'])
- self.assertEquals('stackalytics', commit['module'])
+ def test_process_mail(self):
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+ 'companies': [
+ {'company_name': 'NEC', 'end_date': 0},
+ ]
+ }
+ ],
+ repos=[{"module": "stackalytics"}]
+ )
+
+ processed_commit = list(record_processor_inst.process(
+ generate_emails(
+ author_email='johndoe@gmail.com',
+ author_name='John Doe',
+ subject='[openstack-dev] [Stackalytics] Configuration files')
+ ))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@gmail.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'NEC',
+ 'module': 'stackalytics',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
def test_process_mail_guessed(self):
- runtime_storage_inst = make_runtime_storage(
- repos=[{'module': 'nova'}, {'module': 'neutron'}])
- record_processor_inst = make_record_processor(runtime_storage_inst)
- commit_generator = generate_emails(
- subject='[openstack-dev] [Neutron] [Nova] Integration issue')
- commit = list(record_processor_inst.process(commit_generator))[0]
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+ 'companies': [
+ {'company_name': 'NEC', 'end_date': 0},
+ ]
+ }
+ ],
+ repos=[{'module': 'nova'}, {'module': 'neutron'}]
+ )
- self.assertEquals('neutron', commit['module'])
+ processed_commit = list(record_processor_inst.process(
+ generate_emails(
+ author_email='johndoe@gmail.com',
+ author_name='John Doe',
+ subject='[openstack-dev] [Neutron] [Nova] Integration issue')
+ ))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@gmail.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'NEC',
+ 'module': 'neutron',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
def test_process_mail_guessed_module_in_body_override(self):
- runtime_storage_inst = make_runtime_storage(
- repos=[{'module': 'nova'}, {'module': 'heat'}])
- record_processor_inst = make_record_processor(runtime_storage_inst)
- commit_generator = generate_emails(
- subject='[openstack-dev] [heat] Comments/questions on the',
- module='nova')
- commit = list(record_processor_inst.process(commit_generator))[0]
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+ 'companies': [
+ {'company_name': 'NEC', 'end_date': 0},
+ ]
+ }
+ ],
+ repos=[{'module': 'nova'}, {'module': 'neutron'}]
+ )
- self.assertEquals('heat', commit['module'])
+ processed_commit = list(record_processor_inst.process(
+ generate_emails(
+ author_email='johndoe@gmail.com',
+ author_name='John Doe',
+ module='nova',
+ subject='[openstack-dev] [neutron] Comments/questions on the')
+ ))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@gmail.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'NEC',
+ 'module': 'neutron',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
def test_process_mail_guessed_module_in_body(self):
- runtime_storage_inst = make_runtime_storage(
- repos=[{'module': 'nova'}, {'module': 'heat'}])
- record_processor_inst = make_record_processor(runtime_storage_inst)
- commit_generator = generate_emails(
- subject='[openstack-dev] Comments/questions on the heat',
- module='nova')
- commit = list(record_processor_inst.process(commit_generator))[0]
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+ 'companies': [
+ {'company_name': 'NEC', 'end_date': 0},
+ ]
+ }
+ ],
+ repos=[{'module': 'nova'}, {'module': 'neutron'}]
+ )
- self.assertEquals('nova', commit['module'])
+ processed_commit = list(record_processor_inst.process(
+ generate_emails(
+ author_email='johndoe@gmail.com',
+ author_name='John Doe',
+ module='nova',
+ subject='[openstack-dev] Comments/questions on the')
+ ))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@gmail.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'NEC',
+ 'module': 'nova',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
def test_process_mail_unmatched(self):
- record_processor_inst = make_record_processor()
- commit_generator = generate_emails(
- subject='[openstack-dev] [Photon] Configuration files')
- commit = list(record_processor_inst.process(commit_generator))[0]
+ record_processor_inst = self.make_record_processor(
+ users=[
+ {
+ 'user_id': 'john_doe',
+ 'launchpad_id': 'john_doe',
+ 'user_name': 'John Doe',
+ 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
+ 'companies': [
+ {'company_name': 'NEC', 'end_date': 0},
+ ]
+ }
+ ],
+ repos=[{'module': 'nova'}, {'module': 'neutron'}]
+ )
- self.assertEquals('SuperCompany', commit['company_name'])
- self.assertEquals('john_doe', commit['launchpad_id'])
- self.assertEquals('unknown', commit['module'])
+ processed_commit = list(record_processor_inst.process(
+ generate_emails(
+ author_email='johndoe@gmail.com',
+ author_name='John Doe',
+ subject='[openstack-dev] Comments/questions on the')
+ ))[0]
+
+ expected_commit = {
+ 'launchpad_id': 'john_doe',
+ 'author_email': 'johndoe@gmail.com',
+ 'author_name': 'John Doe',
+ 'company_name': 'NEC',
+ 'module': 'unknown',
+ }
+
+ self.assertRecordsMatch(expected_commit, processed_commit)
def test_get_modules(self):
- record_processor_inst = make_record_processor()
+ record_processor_inst = self.make_record_processor()
with mock.patch('stackalytics.processor.utils.load_repos') as patch:
patch.return_value = [{'module': 'nova'},
{'module': 'python-novaclient'},
@@ -341,15 +687,36 @@ class TestRecordProcessor(testtools.TestCase):
modules = record_processor_inst._get_modules()
self.assertEqual(set(['nova', 'neutron']), set(modules))
+ def assertRecordsMatch(self, expected, actual):
+ for key, value in expected.iteritems():
+ self.assertEquals(value, actual[key],
+ 'Values for key %s do not match' % key)
-# Helpers
+ # Helpers
-def generate_commits(email='johndoe@gmail.com', date=1999999999):
+ def make_record_processor(self, users=None, companies=None, releases=None,
+ repos=None, lp_info=None, lp_user_name=None):
+ rp = record_processor.RecordProcessor(make_runtime_storage(
+ users=users, companies=companies, releases=releases, repos=repos))
+
+ if lp_info is not None:
+ self.lp_profile_by_email.side_effect = (
+ lambda x: lp_info.get(x))
+
+ if lp_user_name is not None:
+ self.lp_profile_by_launchpad_id.side_effect = (
+ lambda x: lp_user_name.get(x))
+
+ return rp
+
+
+def generate_commits(author_name='John Doe', author_email='johndoe@gmail.com',
+ date=1999999999):
yield {
'record_type': 'commit',
'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
- 'author_name': 'John Doe',
- 'author_email': email,
+ 'author_name': author_name,
+ 'author_email': author_email,
'date': date,
'lines_added': 25,
'lines_deleted': 9,
@@ -357,13 +724,13 @@ def generate_commits(email='johndoe@gmail.com', date=1999999999):
}
-def generate_emails(email='johndoe@gmail.com', date=1999999999,
- subject='[openstack-dev]', module=None):
+def generate_emails(author_name='John Doe', author_email='johndoe@gmail.com',
+ date=1999999999, subject='[openstack-dev]', module=None):
yield {
'record_type': 'email',
'message_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
- 'author_name': 'John Doe',
- 'author_email': email,
+ 'author_name': author_name,
+ 'author_email': author_email,
'date': date,
'subject': subject,
'module': module,
@@ -374,9 +741,11 @@ def make_runtime_storage(users=None, companies=None, releases=None,
repos=None):
def get_by_key(collection):
if collection == 'companies':
- return _make_companies(companies or COMPANIES)
+ return _make_companies(companies or [
+ {"company_name": "*independent", "domains": [""]},
+ ])
elif collection == 'users':
- return _make_users(users or USERS)
+ return _make_users(users or [])
elif collection == 'releases':
return releases or RELEASES
elif collection == 'repos':
@@ -389,26 +758,6 @@ def make_runtime_storage(users=None, companies=None, releases=None,
return rs
-def make_record_processor(runtime_storage_inst=None):
- return record_processor.RecordProcessor(runtime_storage_inst or
- make_runtime_storage())
-
-
-def make_user():
- return {
- 'user_id': 'john_doe',
- 'launchpad_id': 'john_doe',
- 'user_name': 'John Doe',
- 'emails': ['johndoe@gmail.com', 'jdoe@super.no'],
- 'companies': [
- {'company_name': '*independent',
- 'end_date': 1234567890},
- {'company_name': 'SuperCompany',
- 'end_date': 0},
- ]
- }
-
-
def _make_users(users):
users_index = {}
for user in users: