# Copyright (c) 2013 Mirantis Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import mock import testtools from stackalytics.processor import record_processor from stackalytics.processor import runtime_storage from stackalytics.processor import utils RELEASES = [ { 'release_name': 'prehistory', 'end_date': utils.date_to_timestamp('2011-Apr-21') }, { 'release_name': 'Diablo', 'end_date': utils.date_to_timestamp('2011-Sep-08') }, { 'release_name': 'Zoo', 'end_date': utils.date_to_timestamp('2035-Sep-08') }, ] REPOS = [ { "branches": ["master"], "module": "stackalytics", "project_type": "stackforge", "uri": "git://github.com/stackforge/stackalytics.git" } ] class TestRecordProcessor(testtools.TestCase): def setUp(self): super(TestRecordProcessor, self).setUp() self.read_json_from_uri_patch = mock.patch( 'stackalytics.processor.utils.read_json_from_uri') self.read_launchpad = self.read_json_from_uri_patch.start() self.lp_profile_by_launchpad_id_patch = mock.patch( 'stackalytics.processor.launchpad_utils.' 'lp_profile_by_launchpad_id') self.lp_profile_by_launchpad_id = ( self.lp_profile_by_launchpad_id_patch.start()) self.lp_profile_by_launchpad_id.return_value = None self.lp_profile_by_email_patch = mock.patch( 'stackalytics.processor.launchpad_utils.lp_profile_by_email') self.lp_profile_by_email = ( self.lp_profile_by_email_patch.start()) self.lp_profile_by_email.return_value = None def tearDown(self): super(TestRecordProcessor, self).tearDown() self.read_json_from_uri_patch.stop() self.lp_profile_by_launchpad_id_patch.stop() self.lp_profile_by_email_patch.stop() # get_company_by_email def test_get_company_by_email_mapped(self): record_processor_inst = self.make_record_processor( companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}] ) email = 'jdoe@ibm.com' res = record_processor_inst._get_company_by_email(email) self.assertEquals('IBM', res) def test_get_company_by_email_with_long_suffix_mapped(self): record_processor_inst = self.make_record_processor( companies=[{'company_name': 'NEC', 'domains': ['nec.co.jp']}] ) email = 'man@mxw.nes.nec.co.jp' res = record_processor_inst._get_company_by_email(email) self.assertEquals('NEC', res) def test_get_company_by_email_with_long_suffix_mapped_2(self): record_processor_inst = self.make_record_processor( companies=[{'company_name': 'NEC', 'domains': ['nec.co.jp', 'nec.com']}] ) email = 'man@mxw.nes.nec.com' res = record_processor_inst._get_company_by_email(email) self.assertEquals('NEC', res) def test_get_company_by_email_not_mapped(self): record_processor_inst = self.make_record_processor() email = 'foo@boo.com' res = record_processor_inst._get_company_by_email(email) self.assertEquals(None, res) # get_lp_info def test_get_lp_info_invalid_email(self): self.read_launchpad.return_value = None record_processor_inst = self.make_record_processor(users=[]) self.assertEquals((None, None), record_processor_inst._get_lp_info('error.root')) # commit processing def test_process_commit_existing_user(self): record_processor_inst = self.make_record_processor( users=[ { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'], 'companies': [ {'company_name': '*independent', 'end_date': 1234567890}, {'company_name': 'NEC', 'end_date': 0}, ] } ]) processed_commit = list(record_processor_inst.process( generate_commits(author_email='johndoe@gmail.com', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@gmail.com', 'author_name': 'John Doe', 'company_name': 'NEC', } self.assertRecordsMatch(expected_commit, processed_commit) def test_process_commit_existing_user_old_job(self): record_processor_inst = self.make_record_processor( users=[ { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'], 'companies': [ {'company_name': '*independent', 'end_date': 1234567890}, {'company_name': 'NEC', 'end_date': 0}, ] } ]) processed_commit = list(record_processor_inst.process( generate_commits(author_email='johndoe@gmail.com', author_name='John Doe', date=1000000000)))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@gmail.com', 'author_name': 'John Doe', 'company_name': '*independent', } self.assertRecordsMatch(expected_commit, processed_commit) def test_process_commit_existing_user_new_email_known_company(self): # User is known to LP, his email is new to us, and maps to other # company. Should return other company instead of those mentioned # in user profile record_processor_inst = self.make_record_processor( users=[ {'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@nec.co.jp'], 'companies': [{'company_name': 'NEC', 'end_date': 0}]} ], companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], lp_info={'johndoe@ibm.com': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_commit = list(record_processor_inst.process( generate_commits(author_email='johndoe@ibm.com', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@ibm.com', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) self.assertIn('johndoe@ibm.com', utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')['emails']) def test_process_commit_existing_user_new_email_unknown_company(self): # User is known to LP, but his email is new to us. Should match # the user and return company from user profile record_processor_inst = self.make_record_processor( users=[ {'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@nec.co.jp'], 'companies': [{'company_name': 'NEC', 'end_date': 0}]} ], companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], lp_info={'johndoe@gmail.com': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_commit = list(record_processor_inst.process( generate_commits(author_email='johndoe@gmail.com', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@gmail.com', 'author_name': 'John Doe', 'company_name': 'NEC', } self.assertRecordsMatch(expected_commit, processed_commit) self.assertIn('johndoe@gmail.com', utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')['emails']) def test_process_commit_existing_user_new_email_known_company_update(self): record_processor_inst = self.make_record_processor( users=[ {'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@gmail.com'], 'companies': [{'company_name': '*independent', 'end_date': 0}]} ], companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], lp_info={'johndoe@ibm.com': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_commit = list(record_processor_inst.process( generate_commits(author_email='johndoe@ibm.com', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@ibm.com', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe') self.assertIn('johndoe@gmail.com', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name'], message='User affiliation should be updated') def test_process_commit_new_user(self): # User is known to LP, but new to us # Should add new user and set company depending on email record_processor_inst = self.make_record_processor( companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], lp_info={'johndoe@ibm.com': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_commit = list(record_processor_inst.process( generate_commits(author_email='johndoe@ibm.com', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@ibm.com', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe') self.assertIn('johndoe@ibm.com', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name']) def test_process_commit_new_user_unknown_to_lb(self): # User is new to us and not known to LP # Should set user name and empty LPid record_processor_inst = self.make_record_processor( companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}]) processed_commit = list(record_processor_inst.process( generate_commits(author_email='johndoe@ibm.com', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': None, 'author_email': 'johndoe@ibm.com', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user( record_processor_inst.runtime_storage_inst, 'johndoe@ibm.com') self.assertIn('johndoe@ibm.com', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name']) self.assertEquals(None, user['launchpad_id']) # process records complex scenarios def test_process_blueprint_one_draft_spawned_lp_doesnt_know_user(self): # In: blueprint record # LP doesn't know user # Out: blueprint-draft record # new user profile created record_processor_inst = self.make_record_processor() processed_records = list(record_processor_inst.process([ {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890} ])) self.assertRecordsMatch( {'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'john_doe', 'company_name': '*independent'}, processed_records[0]) user = utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe') self.assertEquals({ 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'john_doe', 'emails': [], 'companies': [{'company_name': '*independent', 'end_date': 0}] }, user) def test_process_blueprint_one_draft_spawned_lp_knows_user(self): # In: blueprint record # LP knows user # Out: blueprint-draft record # new user profile created, name is taken from LP profile record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_records = list(record_processor_inst.process([ {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890} ])) self.assertRecordsMatch( {'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent'}, processed_records[0]) user = utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe') self.assertEquals({ 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': [], 'companies': [{'company_name': '*independent', 'end_date': 0}] }, user) def test_process_blueprint_then_review(self): record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_records = list(record_processor_inst.process([ {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890}, {'record_type': 'review', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'subject': 'Fix AttributeError in Keypair._add_details()', 'owner': {'name': 'John Doe', 'email': 'john_doe@gmail.com', 'username': 'john_doe'}, 'createdOn': 1379404951, 'module': 'nova'} ])) self.assertRecordsMatch( {'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent'}, processed_records[0]) self.assertRecordsMatch( {'record_type': 'review', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'author_email': 'john_doe@gmail.com', 'company_name': '*independent'}, processed_records[1]) user = {'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['john_doe@gmail.com'], 'companies': [{'company_name': '*independent', 'end_date': 0}]} self.assertEquals(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')) self.assertEquals(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe@gmail.com')) def test_process_blueprint_then_commit(self): record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}}, lp_info={'john_doe@gmail.com': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_records = list(record_processor_inst.process([ {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890}, {'record_type': 'commit', 'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1', 'author_name': 'John Doe', 'author_email': 'john_doe@gmail.com', 'date': 1234567890, 'lines_added': 25, 'lines_deleted': 9, 'release_name': 'havana'} ])) self.assertRecordsMatch( {'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent'}, processed_records[0]) self.assertRecordsMatch( {'record_type': 'commit', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'author_email': 'john_doe@gmail.com', 'company_name': '*independent'}, processed_records[1]) user = {'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['john_doe@gmail.com'], 'companies': [{'company_name': '*independent', 'end_date': 0}]} self.assertEquals(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')) self.assertEquals(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe@gmail.com')) def test_process_review_then_blueprint(self): record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_records = list(record_processor_inst.process([ {'record_type': 'review', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'subject': 'Fix AttributeError in Keypair._add_details()', 'owner': {'name': 'John Doe', 'email': 'john_doe@gmail.com', 'username': 'john_doe'}, 'createdOn': 1379404951, 'module': 'nova'}, {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890} ])) self.assertRecordsMatch( {'record_type': 'review', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'author_email': 'john_doe@gmail.com', 'company_name': '*independent'}, processed_records[0]) self.assertRecordsMatch( {'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent'}, processed_records[1]) user = {'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['john_doe@gmail.com'], 'companies': [{'company_name': '*independent', 'end_date': 0}]} self.assertEquals(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')) self.assertEquals(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe@gmail.com')) # update records def _generate_record_commit(self): yield {'commit_id': u'0afdc64bfd041b03943ceda7849c4443940b6053', 'lines_added': 9, 'module': u'stackalytics', 'record_type': 'commit', 'message': u'Closes bug 1212953\n\nChange-Id: ' u'I33f0f37b6460dc494abf2520dc109c9893ace9e6\n', 'subject': u'Fixed affiliation of Edgar and Sumit', 'loc': 10, 'user_id': u'john_doe', 'primary_key': u'0afdc64bfd041b03943ceda7849c4443940b6053', 'author_email': u'jdoe@super.no', 'company_name': u'SuperCompany', 'record_id': 6, 'lines_deleted': 1, 'week': 2275, 'blueprint_id': None, 'bug_id': u'1212953', 'files_changed': 1, 'author_name': u'John Doe', 'date': 1376737923, 'launchpad_id': u'john_doe', 'branches': set([u'master']), 'change_id': u'I33f0f37b6460dc494abf2520dc109c9893ace9e6', 'release': u'havana'} def test_update_record_no_changes(self): commit_generator = self._generate_record_commit() release_index = {'0afdc64bfd041b03943ceda7849c4443940b6053': 'havana'} record_processor_inst = self.make_record_processor( users=[], companies=[{'company_name': 'SuperCompany', 'domains': ['super.no']}]) updated = list(record_processor_inst.update(commit_generator, release_index)) self.assertEquals(0, len(updated)) # mail processing def test_process_mail(self): record_processor_inst = self.make_record_processor( users=[ { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'], 'companies': [ {'company_name': 'NEC', 'end_date': 0}, ] } ], repos=[{"module": "stackalytics"}] ) processed_commit = list(record_processor_inst.process( generate_emails( author_email='johndoe@gmail.com', author_name='John Doe', subject='[openstack-dev] [Stackalytics] Configuration files') ))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@gmail.com', 'author_name': 'John Doe', 'company_name': 'NEC', 'module': 'stackalytics', } self.assertRecordsMatch(expected_commit, processed_commit) def test_process_mail_guessed(self): record_processor_inst = self.make_record_processor( users=[ { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'], 'companies': [ {'company_name': 'NEC', 'end_date': 0}, ] } ], repos=[{'module': 'nova'}, {'module': 'neutron'}] ) processed_commit = list(record_processor_inst.process( generate_emails( author_email='johndoe@gmail.com', author_name='John Doe', subject='[openstack-dev] [Neutron] [Nova] Integration issue') ))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@gmail.com', 'author_name': 'John Doe', 'company_name': 'NEC', 'module': 'neutron', } self.assertRecordsMatch(expected_commit, processed_commit) def test_process_mail_guessed_module_in_body_override(self): record_processor_inst = self.make_record_processor( users=[ { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'], 'companies': [ {'company_name': 'NEC', 'end_date': 0}, ] } ], repos=[{'module': 'nova'}, {'module': 'neutron'}] ) processed_commit = list(record_processor_inst.process( generate_emails( author_email='johndoe@gmail.com', author_name='John Doe', module='nova', subject='[openstack-dev] [neutron] Comments/questions on the') ))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@gmail.com', 'author_name': 'John Doe', 'company_name': 'NEC', 'module': 'neutron', } self.assertRecordsMatch(expected_commit, processed_commit) def test_process_mail_guessed_module_in_body(self): record_processor_inst = self.make_record_processor( users=[ { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'], 'companies': [ {'company_name': 'NEC', 'end_date': 0}, ] } ], repos=[{'module': 'nova'}, {'module': 'neutron'}] ) processed_commit = list(record_processor_inst.process( generate_emails( author_email='johndoe@gmail.com', author_name='John Doe', module='nova', subject='[openstack-dev] Comments/questions on the') ))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@gmail.com', 'author_name': 'John Doe', 'company_name': 'NEC', 'module': 'nova', } self.assertRecordsMatch(expected_commit, processed_commit) def test_process_mail_unmatched(self): record_processor_inst = self.make_record_processor( users=[ { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'], 'companies': [ {'company_name': 'NEC', 'end_date': 0}, ] } ], repos=[{'module': 'nova'}, {'module': 'neutron'}] ) processed_commit = list(record_processor_inst.process( generate_emails( author_email='johndoe@gmail.com', author_name='John Doe', subject='[openstack-dev] Comments/questions on the') ))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': 'johndoe@gmail.com', 'author_name': 'John Doe', 'company_name': 'NEC', 'module': 'unknown', } self.assertRecordsMatch(expected_commit, processed_commit) def test_get_modules(self): record_processor_inst = self.make_record_processor() with mock.patch('stackalytics.processor.utils.load_repos') as patch: patch.return_value = [{'module': 'nova'}, {'module': 'python-novaclient'}, {'module': 'neutron'}] modules = record_processor_inst._get_modules() self.assertEqual(set(['nova', 'neutron']), set(modules)) def assertRecordsMatch(self, expected, actual): for key, value in expected.iteritems(): self.assertEquals(value, actual[key], 'Values for key %s do not match' % key) # Helpers def make_record_processor(self, users=None, companies=None, releases=None, repos=None, lp_info=None, lp_user_name=None): rp = record_processor.RecordProcessor(make_runtime_storage( users=users, companies=companies, releases=releases, repos=repos)) if lp_info is not None: self.lp_profile_by_email.side_effect = ( lambda x: lp_info.get(x)) if lp_user_name is not None: self.lp_profile_by_launchpad_id.side_effect = ( lambda x: lp_user_name.get(x)) return rp def generate_commits(author_name='John Doe', author_email='johndoe@gmail.com', date=1999999999): yield { 'record_type': 'commit', 'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1', 'author_name': author_name, 'author_email': author_email, 'date': date, 'lines_added': 25, 'lines_deleted': 9, 'release_name': 'havana', } def generate_emails(author_name='John Doe', author_email='johndoe@gmail.com', date=1999999999, subject='[openstack-dev]', module=None): yield { 'record_type': 'email', 'message_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1', 'author_name': author_name, 'author_email': author_email, 'date': date, 'subject': subject, 'module': module, 'body': 'lorem ipsum', } def make_runtime_storage(users=None, companies=None, releases=None, repos=None): runtime_storage_cache = {} def get_by_key(key): if key == 'companies': return _make_companies(companies or [ {"company_name": "*independent", "domains": [""]}, ]) elif key == 'users': return _make_users(users or []) elif key == 'releases': return releases or RELEASES elif key == 'repos': return repos or REPOS else: return runtime_storage_cache.get(key) def set_by_key(key, value): runtime_storage_cache[key] = value rs = mock.Mock(runtime_storage.RuntimeStorage) rs.get_by_key = mock.Mock(side_effect=get_by_key) rs.set_by_key = mock.Mock(side_effect=set_by_key) if users: for user in users: set_by_key('user:%s' % user['user_id'], user) if user.get('launchpad_id'): set_by_key('user:%s' % user['launchpad_id'], user) for email in user.get('emails') or []: set_by_key('user:%s' % email, user) return rs def _make_users(users): users_index = {} for user in users: if 'user_id' in user: users_index[user['user_id']] = user if 'launchpad_id' in user: users_index[user['launchpad_id']] = user for email in user['emails']: users_index[email] = user return users_index def _make_companies(companies): domains_index = {} for company in companies: for domain in company['domains']: domains_index[domain] = company['company_name'] return domains_index