x/stackalytics
Code Issues Proposed changes

Adds support of multiple gerrit_ids

Browse Source 
Previously only one gerrit_id for user_profile was supported and took
part in record_processing. In case of using multiple Gerrits or multiple
gerrit_ids in one Gerrit, while merging profiles all except one
gerrit_ids were rejected.

This commit adds new user gerrit_ids user_profile key containing all
required mapping between multiple Gerrits and gerrit_ids. Each gerrit_id
acts the same way as old gerrit_id during review record processing, that
potentially increases success rate of record identification and
decreses amount of unnecessary launchpad user queries by storing user
with all gerrit_ids as a primary key instead of one.

Also changes to UI added to represent multiple Gerrit accounts.

Change-Id: Id20c7ed46cd409946b9a5247a94331860895f859
This commit is contained in:
Andrii Ostapenko 2019-03-11 20:29:55 -05:00 committed by Roman Gorshunov
parent 432f4edc08
commit 8b5345b458
6 changed files with 229 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
stackalytics/dashboard/templates/_macros/user_profile.html
View File

@ -32,8 +32,16 @@
{%if launchpad_id %} {%if launchpad_id %}
<div>Launchpad: <a href="https://launchpad.net/~${launchpad_id}" target="_blank">${launchpad_id}</a></div> <div>Launchpad: <a href="https://launchpad.net/~${launchpad_id}" target="_blank">${launchpad_id}</a></div>
{%/if%} {%/if%}
{%if gerrit_id %} {%if gerrit_ids %}
<div>Gerrit: <a href="https://review.openstack.org/#/q/owner:${gerrit_id},n,z" target="_blank">${gerrit_id}</a></div> <div>Gerrit:</div>
{%each(hostname, ids) gerrit_ids %}
<div style="text-indent: 10px">
${hostname}:
{%each(i, id) ids%}
<a href="http://${hostname}/#/q/owner:${id},n,z" target="_blank">${id}</a>
{%/each%}
</div>
{%/each%}
{%/if%} {%/if%}
{%if github_id %} {%if github_id %}
<div>Github: <a href="https://github.com/${github_id}" target="_blank">${github_id}</a></div> <div>Github: <a href="https://github.com/${github_id}" target="_blank">${github_id}</a></div>

7
stackalytics/processor/dump.py
View File

@ -139,8 +139,11 @@ def export_data(memcached_inst, fd):
user), fd) user), fd)
if user.get('launchpad_id'): if user.get('launchpad_id'):
pickle.dump(('user:%s' % user['launchpad_id'], user), fd) pickle.dump(('user:%s' % user['launchpad_id'], user), fd)
if user.get('gerrit_id'): for hostname, ids in user.get('gerrit_ids', {}).items():
pickle.dump(('user:gerrit:%s' % user['gerrit_id'], user), fd) for gerrit_id in ids:
pickle.dump(('user:gerrit:%s:%s' % (hostname, gerrit_id),
user),
fd)
if user.get('member_id'): if user.get('member_id'):
pickle.dump(('user:member:%s' % user['member_id'], user), fd) pickle.dump(('user:member:%s' % user['member_id'], user), fd)
for email in user.get('emails') or []: for email in user.get('emails') or []:

11
stackalytics/processor/main.py
View File

@ -122,6 +122,7 @@ def _process_repo_reviews(repo, runtime_storage_inst, record_processor_inst):
rcs_inst.setup(key_filename=repo['key_filename'], rcs_inst.setup(key_filename=repo['key_filename'],
username=repo['ssh_username'], username=repo['ssh_username'],
gerrit_retry=CONF.gerrit_retry) gerrit_retry=CONF.gerrit_retry)
gerrit_hostname = rcs.get_socket_tuple_from_uri(repo['gerrit_uri'])[0]
for branch in _get_repo_branches(repo): for branch in _get_repo_branches(repo):
LOG.info('Processing reviews for repo: %s, branch: %s', LOG.info('Processing reviews for repo: %s, branch: %s',
@ -139,7 +140,11 @@ def _process_repo_reviews(repo, runtime_storage_inst, record_processor_inst):
rcs_inst.log(repo, branch, last_retrieval_time, status='abandoned', rcs_inst.log(repo, branch, last_retrieval_time, status='abandoned',
grab_comments=True), ) grab_comments=True), )
review_iterator_typed = _record_typer(review_iterator, 'review') review_iterator_with_gerrit = _param_adder(
review_iterator, 'gerrit_hostname', gerrit_hostname
)
review_iterator_typed = _record_typer(review_iterator_with_gerrit,
'review')
processed_review_iterator = record_processor_inst.process( processed_review_iterator = record_processor_inst.process(
review_iterator_typed) review_iterator_typed)
@ -153,9 +158,9 @@ def _process_repo_reviews(repo, runtime_storage_inst, record_processor_inst):
def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst): def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst):
vcs_inst = vcs.get_vcs(repo, CONF.sources_root) vcs_inst = vcs.get_vcs(repo, CONF.sources_root)
vcs_inst.fetch() vcs_inst.fetch()
gerrit_hostname, _ = rcs.get_socket_tuple_from_uri( gerrit_hostname = rcs.get_socket_tuple_from_uri(
repo.get('gerrit_uri', CONF.review_uri) repo.get('gerrit_uri', CONF.review_uri)
) )[0]
for branch in _get_repo_branches(repo): for branch in _get_repo_branches(repo):
LOG.info('Processing commits in repo: %s, branch: %s', LOG.info('Processing commits in repo: %s, branch: %s',

8
stackalytics/processor/record_processor.py
View File

@ -99,9 +99,11 @@ class RecordProcessor(object):
user_name = lp_user_name user_name = lp_user_name
gerrit_id = record.get('gerrit_id') gerrit_id = record.get('gerrit_id')
gerrit_tuple = None
if gerrit_id: if gerrit_id:
gerrit_tuple = (record['gerrit_hostname'], gerrit_id)
user_g = user_processor.load_user( user_g = user_processor.load_user(
self.runtime_storage_inst, gerrit_id=gerrit_id) or {} self.runtime_storage_inst, gerrit_tuple=gerrit_tuple) or {}
if (self._need_to_fetch_launchpad() and (not user_g) and if (self._need_to_fetch_launchpad() and (not user_g) and
(not launchpad_id) and (not user_e.get('launchpad_id'))): (not launchpad_id) and (not user_e.get('launchpad_id'))):
# query LP # query LP
@ -135,7 +137,7 @@ class RecordProcessor(object):
return user_e return user_e
user = user_processor.create_user( user = user_processor.create_user(
self.domains_index, launchpad_id, email, gerrit_id, zanata_id, self.domains_index, launchpad_id, email, gerrit_tuple, zanata_id,
user_name) user_name)
if user_e or user_l or user_g or user_z: if user_e or user_l or user_g or user_z:
@ -262,6 +264,7 @@ class RecordProcessor(object):
or 'Anonymous Coward') or 'Anonymous Coward')
if uploader.get('email'): if uploader.get('email'):
patch_record['author_email'] = uploader['email'].lower() patch_record['author_email'] = uploader['email'].lower()
patch_record['gerrit_hostname'] = review['gerrit_hostname']
patch_record['module'] = review['module'] patch_record['module'] = review['module']
patch_record['branch'] = review['branch'] patch_record['branch'] = review['branch']
patch_record['review_id'] = review['id'] patch_record['review_id'] = review['id']
@ -286,6 +289,7 @@ class RecordProcessor(object):
mark['branch'] = review['branch'] mark['branch'] = review['branch']
mark['review_id'] = review['id'] mark['review_id'] = review['id']
mark['patch'] = int(patch['number']) mark['patch'] = int(patch['number'])
mark['gerrit_hostname'] = review['gerrit_hostname']
if reviewer['username'] == patch['uploader'].get('username'): if reviewer['username'] == patch['uploader'].get('username'):
# reviewer is the same as author of the patch # reviewer is the same as author of the patch

70
stackalytics/processor/user_processor.py
View File

@ -23,12 +23,12 @@ INDEPENDENT = '*independent'
ROBOTS = '*robots' ROBOTS = '*robots'
def make_user_id(emails=None, launchpad_id=None, gerrit_id=None, def make_user_id(emails=None, launchpad_id=None, gerrit_tuple=None,
member_id=None, github_id=None, zanata_id=None): member_id=None, github_id=None, zanata_id=None):
if launchpad_id or emails: if launchpad_id or emails:
return launchpad_id or emails[0] return launchpad_id or emails[0]
if gerrit_id: if gerrit_tuple:
return 'gerrit:%s' % gerrit_id return 'gerrit:%s:%s' % gerrit_tuple
if member_id: if member_id:
return 'member:%s' % member_id return 'member:%s' % member_id
if github_id: if github_id:
@ -48,9 +48,10 @@ def store_user(runtime_storage_inst, user):
runtime_storage_inst.set_by_key('user:%s' % user['user_id'], user) runtime_storage_inst.set_by_key('user:%s' % user['user_id'], user)
if user.get('launchpad_id'): if user.get('launchpad_id'):
runtime_storage_inst.set_by_key('user:%s' % user['launchpad_id'], user) runtime_storage_inst.set_by_key('user:%s' % user['launchpad_id'], user)
if user.get('gerrit_id'): for hostname, ids in user.get('gerrit_ids', {}).items():
runtime_storage_inst.set_by_key('user:gerrit:%s' % user['gerrit_id'], for gerrit_id in ids:
user) runtime_storage_inst.set_by_key(
'user:gerrit:%s:%s' % (hostname, gerrit_id), user)
if user.get('github_id'): if user.get('github_id'):
runtime_storage_inst.set_by_key('user:github:%s' % user['github_id'], runtime_storage_inst.set_by_key('user:github:%s' % user['github_id'],
user) user)
@ -62,10 +63,10 @@ def store_user(runtime_storage_inst, user):
def load_user(runtime_storage_inst, seq=None, user_id=None, email=None, def load_user(runtime_storage_inst, seq=None, user_id=None, email=None,
launchpad_id=None, gerrit_id=None, member_id=None, launchpad_id=None, gerrit_tuple=None, member_id=None,
github_id=None, zanata_id=None): github_id=None, zanata_id=None):
key = make_user_id(gerrit_id=gerrit_id, member_id=member_id, key = make_user_id(gerrit_tuple=gerrit_tuple, member_id=member_id,
github_id=github_id, zanata_id=zanata_id) github_id=github_id, zanata_id=zanata_id)
if not key: if not key:
key = seq or user_id or launchpad_id or email key = seq or user_id or launchpad_id or email
@ -85,8 +86,14 @@ def update_user_profile(stored_user, user):
if stored_user: if stored_user:
updated_user = copy.deepcopy(stored_user) updated_user = copy.deepcopy(stored_user)
updated_user.update(user) updated_user.update(user)
updated_user['emails'] = list(set(stored_user.get('emails', [])) | updated_user['emails'] = sorted(
set(user.get('emails', []))) list(set(stored_user.get('emails', [])) |
set(user.get('emails', [])))
)
gerrit_ids = _merge_gerrit_ids([stored_user, user])
if gerrit_ids:
updated_user['gerrit_ids'] = gerrit_ids
else: else:
updated_user = copy.deepcopy(user) updated_user = copy.deepcopy(user)
updated_user['static'] = True updated_user['static'] = True
@ -123,14 +130,15 @@ def get_company_by_email(domains_index, email):
return None return None
def create_user(domains_index, launchpad_id, email, gerrit_id, zanata_id, def create_user(domains_index, launchpad_id, email, gerrit_tuple, zanata_id,
user_name): user_name):
company = get_company_by_email(domains_index, email) or INDEPENDENT company = get_company_by_email(domains_index, email) or INDEPENDENT
emails = [email] if email else [] emails = [email] if email else []
user = { user = {
'user_id': make_user_id( 'user_id': make_user_id(
emails=emails, launchpad_id=launchpad_id, gerrit_id=gerrit_id, emails=emails, launchpad_id=launchpad_id,
gerrit_tuple=gerrit_tuple,
zanata_id=zanata_id), zanata_id=zanata_id),
'launchpad_id': launchpad_id, 'launchpad_id': launchpad_id,
'user_name': user_name or '', 'user_name': user_name or '',
@ -141,8 +149,10 @@ def create_user(domains_index, launchpad_id, email, gerrit_id, zanata_id,
'emails': emails, 'emails': emails,
} }
if gerrit_id: if gerrit_tuple:
user['gerrit_id'] = gerrit_id user['gerrit_ids'] = {
gerrit_tuple[0]: [gerrit_tuple[1]]
}
if zanata_id: if zanata_id:
user['zanata_id'] = zanata_id user['zanata_id'] = zanata_id
@ -170,6 +180,20 @@ def update_user_affiliation(domains_index, user):
break break
def _merge_gerrit_ids(users):
gerrit_ids = {}
hostnames = set()
for user in users:
hostnames.update(set(user.get('gerrit_ids', {}).keys()))
for hostname in hostnames:
ids = set()
for user in users:
ids |= set(user.get('gerrit_ids', {}).get(hostname, []))
if ids:
gerrit_ids[hostname] = sorted(list(ids))
return gerrit_ids
def merge_user_profiles(domains_index, user_profiles): def merge_user_profiles(domains_index, user_profiles):
"""Merge user profiles into one """Merge user profiles into one
@ -182,23 +206,18 @@ def merge_user_profiles(domains_index, user_profiles):
""" """
LOG.debug('Merge profiles: %s', user_profiles) LOG.debug('Merge profiles: %s', user_profiles)
# check of there are more than 1 launchpad_id nor gerrit_id # check of there are more than 1 launchpad_id
lp_ids = set(u.get('launchpad_id') for u in user_profiles lp_ids = set(u.get('launchpad_id') for u in user_profiles
if u.get('launchpad_id')) if u.get('launchpad_id'))
if len(lp_ids) > 1: if len(lp_ids) > 1:
LOG.debug('Ambiguous launchpad ids: %s on profiles: %s', LOG.debug('Ambiguous launchpad ids: %s on profiles: %s',
lp_ids, user_profiles) lp_ids, user_profiles)
g_ids = set(u.get('gerrit_id') for u in user_profiles
if u.get('gerrit_id'))
if len(g_ids) > 1:
LOG.debug('Ambiguous gerrit ids: %s on profiles: %s',
g_ids, user_profiles)
merged_user = {} # merged user profile merged_user = {} # merged user profile
# collect ordinary fields # collect ordinary fields
for key in ['seq', 'user_name', 'user_id', 'gerrit_id', 'github_id', for key in ['seq', 'user_name', 'user_id', 'github_id', 'launchpad_id',
'launchpad_id', 'companies', 'static', 'zanata_id']: 'companies', 'static', 'zanata_id']:
value = next((v.get(key) for v in user_profiles if v.get(key)), value = next((v.get(key) for v in user_profiles if v.get(key)),
None) None)
if value: if value:
@ -218,9 +237,12 @@ def merge_user_profiles(domains_index, user_profiles):
for u in user_profiles: for u in user_profiles:
emails |= set(u.get('emails', [])) emails |= set(u.get('emails', []))
core_in |= set(u.get('core', [])) core_in |= set(u.get('core', []))
merged_user['emails'] = list(emails) merged_user['emails'] = sorted(list(emails))
if core_in: if core_in:
merged_user['core'] = list(core_in) merged_user['core'] = sorted(list(core_in))
gerrit_ids = _merge_gerrit_ids(user_profiles)
if gerrit_ids:
merged_user['gerrit_ids'] = gerrit_ids
# merge companies # merge companies
merged_companies = merged_user['companies'] merged_companies = merged_user['companies']

163
stackalytics/tests/unit/test_record_processor.py
View File

@ -398,6 +398,7 @@ class TestRecordProcessor(testtools.TestCase):
processed_review = list(record_processor_inst.process([ processed_review = list(record_processor_inst.process([
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'johndoe@ibm.com', 'email': 'johndoe@ibm.com',
@ -416,7 +417,8 @@ class TestRecordProcessor(testtools.TestCase):
self.assertRecordsMatch(expected_review, processed_review) self.assertRecordsMatch(expected_review, processed_review)
user = user_processor.load_user( user = user_processor.load_user(
record_processor_inst.runtime_storage_inst, user_id='john_doe') record_processor_inst.runtime_storage_inst, user_id='john_doe')
self.assertEqual('John_Doe', user['gerrit_id']) self.assertEqual('John_Doe',
user['gerrit_ids']['review.openstack.org'][0])
def test_process_review_without_name(self): def test_process_review_without_name(self):
record_processor_inst = self.make_record_processor() record_processor_inst = self.make_record_processor()
@ -424,6 +426,7 @@ class TestRecordProcessor(testtools.TestCase):
records = list(record_processor_inst.process([ records = list(record_processor_inst.process([
{ {
'record_type': 'review', 'record_type': 'review',
'gerrit_hostname': 'review.openstack.org',
'module': 'sandbox', 'module': 'sandbox',
"project": "openstack-dev/sandbox", "project": "openstack-dev/sandbox",
"branch": "master", "branch": "master",
@ -661,6 +664,7 @@ class TestRecordProcessor(testtools.TestCase):
'date_created': 1234567890}, 'date_created': 1234567890},
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@gmail.com', 'email': 'john_doe@gmail.com',
@ -687,7 +691,7 @@ class TestRecordProcessor(testtools.TestCase):
user = {'seq': 1, user = {'seq': 1,
'user_id': 'john_doe', 'user_id': 'john_doe',
'launchpad_id': 'john_doe', 'launchpad_id': 'john_doe',
'gerrit_id': 'john_doe', 'gerrit_ids': {'review.openstack.org': ['john_doe']},
'user_name': 'John Doe', 'user_name': 'John Doe',
'emails': ['john_doe@gmail.com'], 'emails': ['john_doe@gmail.com'],
'companies': [{'company_name': '*independent', 'end_date': 0}]} 'companies': [{'company_name': '*independent', 'end_date': 0}]}
@ -757,6 +761,7 @@ class TestRecordProcessor(testtools.TestCase):
processed_records = list(record_processor_inst.process([ processed_records = list(record_processor_inst.process([
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@gmail.com', 'email': 'john_doe@gmail.com',
@ -788,7 +793,7 @@ class TestRecordProcessor(testtools.TestCase):
user = {'seq': 1, user = {'seq': 1,
'user_id': 'john_doe', 'user_id': 'john_doe',
'launchpad_id': 'john_doe', 'launchpad_id': 'john_doe',
'gerrit_id': 'john_doe', 'gerrit_ids': {'review.openstack.org': ['john_doe']},
'user_name': 'John Doe', 'user_name': 'John Doe',
'emails': ['john_doe@gmail.com'], 'emails': ['john_doe@gmail.com'],
'companies': [{'company_name': '*independent', 'end_date': 0}]} 'companies': [{'company_name': '*independent', 'end_date': 0}]}
@ -867,6 +872,7 @@ class TestRecordProcessor(testtools.TestCase):
'date': 1234567890}, 'date': 1234567890},
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@gmail.com', 'email': 'john_doe@gmail.com',
@ -877,7 +883,7 @@ class TestRecordProcessor(testtools.TestCase):
user = {'seq': 1, user = {'seq': 1,
'user_id': 'john_doe@gmail.com', 'user_id': 'john_doe@gmail.com',
'gerrit_id': 'john_doe', 'gerrit_ids': {'review.openstack.org': ['john_doe']},
'user_name': 'John Doe', 'user_name': 'John Doe',
'emails': ['john_doe@gmail.com'], 'emails': ['john_doe@gmail.com'],
'companies': [{'company_name': '*independent', 'end_date': 0}]} 'companies': [{'company_name': '*independent', 'end_date': 0}]}
@ -886,7 +892,7 @@ class TestRecordProcessor(testtools.TestCase):
email='john_doe@gmail.com')) email='john_doe@gmail.com'))
self.assertEqual(user, user_processor.load_user( self.assertEqual(user, user_processor.load_user(
record_processor_inst.runtime_storage_inst, record_processor_inst.runtime_storage_inst,
gerrit_id='john_doe')) gerrit_tuple=('review.openstack.org', 'john_doe')))
def test_process_email_then_review_gerrit_id_same_as_launchpad_id(self): def test_process_email_then_review_gerrit_id_same_as_launchpad_id(self):
# it is expected that the user profile will contain email, LP id and # it is expected that the user profile will contain email, LP id and
@ -905,6 +911,7 @@ class TestRecordProcessor(testtools.TestCase):
'date': 1234567890}, 'date': 1234567890},
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@gmail.com', 'email': 'john_doe@gmail.com',
@ -916,7 +923,7 @@ class TestRecordProcessor(testtools.TestCase):
user = {'seq': 1, user = {'seq': 1,
'user_id': 'john_doe', 'user_id': 'john_doe',
'launchpad_id': 'john_doe', 'launchpad_id': 'john_doe',
'gerrit_id': 'john_doe', 'gerrit_ids': {'review.openstack.org': ['john_doe']},
'user_name': 'John Doe', 'user_name': 'John Doe',
'emails': ['john_doe@gmail.com'], 'emails': ['john_doe@gmail.com'],
'companies': [{'company_name': '*independent', 'end_date': 0}]} 'companies': [{'company_name': '*independent', 'end_date': 0}]}
@ -928,7 +935,7 @@ class TestRecordProcessor(testtools.TestCase):
user_id='john_doe')) user_id='john_doe'))
self.assertEqual(user, user_processor.load_user( self.assertEqual(user, user_processor.load_user(
record_processor_inst.runtime_storage_inst, record_processor_inst.runtime_storage_inst,
gerrit_id='john_doe')) gerrit_tuple=('review.openstack.org', 'john_doe')))
def test_process_commit_then_review_with_different_email(self): def test_process_commit_then_review_with_different_email(self):
record_processor_inst = self.make_record_processor( record_processor_inst = self.make_record_processor(
@ -946,6 +953,7 @@ class TestRecordProcessor(testtools.TestCase):
'release_name': 'havana'}, 'release_name': 'havana'},
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'Bill Smith', 'email': 'bill@smith.to', 'owner': {'name': 'Bill Smith', 'email': 'bill@smith.to',
'username': 'bsmith'}, 'username': 'bsmith'},
@ -966,6 +974,7 @@ class TestRecordProcessor(testtools.TestCase):
user = {'seq': 1, user = {'seq': 1,
'user_id': 'john_doe', 'user_id': 'john_doe',
'launchpad_id': 'john_doe', 'launchpad_id': 'john_doe',
'gerrit_ids': {'review.openstack.org': ['john_doe']},
'user_name': 'John Doe', 'user_name': 'John Doe',
'emails': ['john_doe@ibm.com', 'john_doe@gmail.com'], 'emails': ['john_doe@ibm.com', 'john_doe@gmail.com'],
'companies': [{'company_name': 'IBM', 'end_date': 0}]} 'companies': [{'company_name': 'IBM', 'end_date': 0}]}
@ -1002,6 +1011,7 @@ class TestRecordProcessor(testtools.TestCase):
'date': 1234567890}, 'date': 1234567890},
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@ibm.com', 'email': 'john_doe@ibm.com',
@ -1015,7 +1025,7 @@ class TestRecordProcessor(testtools.TestCase):
user = {'seq': 2, user = {'seq': 2,
'user_id': 'john_doe', 'user_id': 'john_doe',
'launchpad_id': 'john_doe', 'launchpad_id': 'john_doe',
'gerrit_id': 'john_doe', 'gerrit_ids': {'review.openstack.org': ['john_doe']},
'user_name': 'John Doe', 'user_name': 'John Doe',
'emails': ['john_doe@ibm.com'], 'emails': ['john_doe@ibm.com'],
'companies': [{'company_name': 'IBM', 'end_date': 0}]} 'companies': [{'company_name': 'IBM', 'end_date': 0}]}
@ -1030,7 +1040,138 @@ class TestRecordProcessor(testtools.TestCase):
self.assertEqual(user, user_processor.load_user( self.assertEqual(user, user_processor.load_user(
runtime_storage_inst, email='john_doe@ibm.com')) runtime_storage_inst, email='john_doe@ibm.com'))
self.assertEqual(user, user_processor.load_user( self.assertEqual(user, user_processor.load_user(
runtime_storage_inst, gerrit_id='john_doe')) runtime_storage_inst,
gerrit_tuple=('review.openstack.org', 'john_doe')))
# all records should have the same user_id and company name
for record in runtime_storage_inst.get_all_records():
self.assertEqual('john_doe', record['user_id'],
message='Record %s' % record['primary_key'])
self.assertEqual('IBM', record['company_name'],
message='Record %s' % record['primary_key'])
def test_process_reviews_two_gerrits(self):
record_processor_inst = self.make_record_processor(
lp_user_name={
'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}
},
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
)
runtime_storage_inst = record_processor_inst.runtime_storage_inst
runtime_storage_inst.set_records(record_processor_inst.process([
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': 'john_doe@ibm.com',
'username': 'john_doe'},
'createdOn': 1379404951,
'module': 'nova', 'branch': 'master'},
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3f',
'gerrit_hostname': 'review.gerrithub.io',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': 'john_doe@ibm.com',
'username': 'john_doe'},
'createdOn': 1379404951,
'module': 'cinder', 'branch': 'master'}
]))
user = {'seq': 1,
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'gerrit_ids': {
'review.openstack.org': ['john_doe'],
'review.gerrithub.io': ['john_doe'],
},
'user_name': 'John Doe',
'emails': ['john_doe@ibm.com'],
'companies': [{'company_name': 'IBM', 'end_date': 0}]}
runtime_storage_inst = record_processor_inst.runtime_storage_inst
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst, user_id='john_doe'))
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst, email='john_doe@ibm.com'))
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst,
gerrit_tuple=('review.openstack.org', 'john_doe')))
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst,
gerrit_tuple=('review.gerrithub.io', 'john_doe')))
# all records should have the same user_id and company name
for record in runtime_storage_inst.get_all_records():
self.assertEqual('john_doe', record['user_id'],
message='Record %s' % record['primary_key'])
self.assertEqual('IBM', record['company_name'],
message='Record %s' % record['primary_key'])
def test_process_reviews_two_ids_in_one_gerrit(self):
record_processor_inst = self.make_record_processor(
lp_user_name={
'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}
},
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
)
runtime_storage_inst = record_processor_inst.runtime_storage_inst
runtime_storage_inst.set_records(record_processor_inst.process([
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': 'john_doe@ibm.com',
'username': 'john_doe'},
'createdOn': 1379404951,
'module': 'nova', 'branch': 'master'},
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3f',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': 'john_doe@ibm.com',
'username': 'johnathan_doe'},
'createdOn': 1379404951,
'module': 'nova', 'branch': 'master'},
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3i',
'gerrit_hostname': 'review.gerrithub.io',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': 'john_doe@ibm.com',
'username': 'johnathan_doe'},
'createdOn': 1379404951,
'module': 'cinder', 'branch': 'master'}
]))
user = {'seq': 1,
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'gerrit_ids': {
'review.openstack.org': ['john_doe', 'johnathan_doe'],
'review.gerrithub.io': ['johnathan_doe'],
},
'user_name': 'John Doe',
'emails': ['john_doe@ibm.com'],
'companies': [{'company_name': 'IBM', 'end_date': 0}]}
runtime_storage_inst = record_processor_inst.runtime_storage_inst
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst, user_id='john_doe'))
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst, email='john_doe@ibm.com'))
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst,
gerrit_tuple=('review.openstack.org', 'john_doe')))
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst,
gerrit_tuple=('review.openstack.org', 'johnathan_doe')))
self.assertEqual(user, user_processor.load_user(
runtime_storage_inst,
gerrit_tuple=('review.gerrithub.io', 'johnathan_doe')))
# all records should have the same user_id and company name # all records should have the same user_id and company name
for record in runtime_storage_inst.get_all_records(): for record in runtime_storage_inst.get_all_records():
@ -1053,6 +1194,7 @@ class TestRecordProcessor(testtools.TestCase):
runtime_storage_inst.set_records(record_processor_inst.process([ runtime_storage_inst.set_records(record_processor_inst.process([
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@ibm.com', 'email': 'john_doe@ibm.com',
@ -1228,6 +1370,7 @@ class TestRecordProcessor(testtools.TestCase):
runtime_storage_inst.set_records(record_processor_inst.process([ runtime_storage_inst.set_records(record_processor_inst.process([
{'record_type': 'review', {'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@ibm.com', 'email': 'john_doe@ibm.com',
@ -1328,6 +1471,7 @@ class TestRecordProcessor(testtools.TestCase):
'release_name': 'havana'}, 'release_name': 'havana'},
{'record_type': 'review', {'record_type': 'review',
'id': 'I104573', 'id': 'I104573',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@gmail.com', 'email': 'john_doe@gmail.com',
@ -1361,6 +1505,7 @@ class TestRecordProcessor(testtools.TestCase):
'release_name': 'havana'}, 'release_name': 'havana'},
{'record_type': 'review', {'record_type': 'review',
'id': 'I104573', 'id': 'I104573',
'gerrit_hostname': 'review.openstack.org',
'subject': 'Fix AttributeError in Keypair._add_details()', 'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe', 'owner': {'name': 'John Doe',
'email': 'john_doe@gmail.com', 'email': 'john_doe@gmail.com',