x/stackalytics
Code Issues Proposed changes

Launchpad lib is replaced by direct request via urllib

Browse Source 
Launchpad lib is used to find user by his email. The same can be done by
accessing LP API directly. This operation doesn't require login to LP and
works much faster.

Closes bug 1213071

Change-Id: I3a6add2a78e2f493c721840a8c8b9041d8e49654
This commit is contained in:
Ilya Shakhat
2013-08-16 23:27:36 +04:00
parent 0889997281
commit b721d95d87
6 changed files with 61 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@@ -2,7 +2,6 @@ d2to1>=0.2.10,<0.3
Flask>=0.9 Flask>=0.9
Flask-Gravatar Flask-Gravatar
iso8601 iso8601
launchpadlib
oslo.config oslo.config
paramiko>=1.8.0 paramiko>=1.8.0
pbr>=0.5.16,<0.6 pbr>=0.5.16,<0.6

28
stackalytics/processor/main.py
View File

@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import json
import urllib import urllib
from oslo.config import cfg from oslo.config import cfg
@@ -26,6 +25,7 @@ from stackalytics.processor import default_data_processor
from stackalytics.processor import rcs from stackalytics.processor import rcs
from stackalytics.processor import record_processor from stackalytics.processor import record_processor
from stackalytics.processor import runtime_storage from stackalytics.processor import runtime_storage
from stackalytics.processor import utils
from stackalytics.processor import vcs from stackalytics.processor import vcs
@@ -147,12 +147,13 @@ def update_repos(runtime_storage_inst):
def apply_corrections(uri, runtime_storage_inst): def apply_corrections(uri, runtime_storage_inst):
LOG.info('Applying corrections from uri %s', uri) LOG.info('Applying corrections from uri %s', uri)
corrections_fd = urllib.urlopen(uri) corrections = utils.read_json_from_uri(uri)
raw = corrections_fd.read() if not corrections:
corrections_fd.close() LOG.error('Unable to read corrections from uri: %s', uri)
corrections = json.loads(raw)['corrections'] return
valid_corrections = [] valid_corrections = []
for c in corrections: for c in corrections['corrections']:
if 'primary_key' in c: if 'primary_key' in c:
valid_corrections.append(c) valid_corrections.append(c)
else: else:
@@ -160,16 +161,6 @@ def apply_corrections(uri, runtime_storage_inst):
runtime_storage_inst.apply_corrections(valid_corrections) runtime_storage_inst.apply_corrections(valid_corrections)
def _read_default_data(uri):
try:
fd = urllib.urlopen(uri)
raw = fd.read()
fd.close()
return json.loads(raw)
except Exception as e:
LOG.error('Error while reading config: %s' % e)
def main(): def main():
# init conf and logging # init conf and logging
conf = cfg.CONF conf = cfg.CONF
@@ -183,7 +174,10 @@ def main():
runtime_storage_inst = runtime_storage.get_runtime_storage( runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri) cfg.CONF.runtime_storage_uri)
default_data = _read_default_data(cfg.CONF.default_data_uri) default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
default_data_processor.process(runtime_storage_inst, default_data_processor.process(runtime_storage_inst,
default_data, default_data,
cfg.CONF.sources_root) cfg.CONF.sources_root)

2
stackalytics/processor/rcs.py
View File

@@ -77,8 +77,6 @@ class Gerrit(Rcs):
def _get_cmd(self, project_organization, module, branch, sort_key, def _get_cmd(self, project_organization, module, branch, sort_key,
limit=PAGE_LIMIT): limit=PAGE_LIMIT):
# This command matches project by substring, not strict
# See https://bugs.launchpad.net/stackalytics/+bug/1212647
cmd = ('gerrit query --all-approvals --patch-sets --format JSON ' cmd = ('gerrit query --all-approvals --patch-sets --format JSON '
'project:\'%(ogn)s/%(module)s\' branch:%(branch)s ' 'project:\'%(ogn)s/%(module)s\' branch:%(branch)s '
'limit:%(limit)s' % 'limit:%(limit)s' %

14
stackalytics/processor/record_processor.py
View File

@@ -16,7 +16,6 @@
import bisect import bisect
import re import re
from launchpadlib import launchpad
from stackalytics.openstack.common import log as logging from stackalytics.openstack.common import log as logging
from stackalytics.processor import normalizer from stackalytics.processor import normalizer
from stackalytics.processor import utils from stackalytics.processor import utils
@@ -78,18 +77,15 @@ class RecordProcessor(object):
LOG.debug('User email is not valid %s' % email) LOG.debug('User email is not valid %s' % email)
else: else:
LOG.debug('Lookup user email %s at Launchpad' % email) LOG.debug('Lookup user email %s at Launchpad' % email)
lp = launchpad.Launchpad.login_anonymously('stackalytics', uri = ('https://api.launchpad.net/1.0/people/?'
'production') 'ws.op=getByEmail&email=%s' % email)
try: lp_profile = utils.read_json_from_uri(uri)
lp_profile = lp.people.getByEmail(email=email)
except Exception as error:
LOG.warn('Lookup of email %s failed %s', email, error.message)
if not lp_profile: if not lp_profile:
LOG.debug('User with email %s not found', email) LOG.debug('User with email %s not found', email)
return None, None return None, None
return lp_profile.name, lp_profile.display_name return lp_profile['name'], lp_profile['display_name']
def _get_independent(self): def _get_independent(self):
return self.domains_index[''] return self.domains_index['']
@@ -144,6 +140,7 @@ class RecordProcessor(object):
self._update_record_and_user(record) self._update_record_and_user(record)
if record['company_name'] != '*robots':
yield record yield record
def _spawn_review(self, record): def _spawn_review(self, record):
@@ -162,7 +159,6 @@ class RecordProcessor(object):
self._update_record_and_user(review) self._update_record_and_user(review)
if record['company_name'] != '*robots':
yield review yield review
def _spawn_marks(self, record): def _spawn_marks(self, record):

17
stackalytics/processor/utils.py
View File

@@ -14,7 +14,14 @@
# limitations under the License. # limitations under the License.
import datetime import datetime
import json
import time import time
import urllib
from stackalytics.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def date_to_timestamp(d): def date_to_timestamp(d):
@@ -33,3 +40,13 @@ def week_to_date(week):
timestamp = week * 7 * 24 * 3600 + 3 * 24 * 3600 timestamp = week * 7 * 24 * 3600 + 3 * 24 * 3600
return (datetime.datetime.fromtimestamp(timestamp). return (datetime.datetime.fromtimestamp(timestamp).
strftime('%Y-%m-%d %H:%M:%S')) strftime('%Y-%m-%d %H:%M:%S'))
def read_json_from_uri(uri):
try:
fd = urllib.urlopen(uri)
raw = fd.read()
fd.close()
return json.loads(raw)
except Exception as e:
LOG.warn('Error while reading uri: %s' % e)

80
tests/unit/test_record_processor.py
View File

@@ -13,9 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from launchpadlib import launchpad
import mock import mock
from oslo.config import cfg
import testtools import testtools
from stackalytics.processor import default_data_processor from stackalytics.processor import default_data_processor
@@ -24,6 +22,9 @@ from stackalytics.processor import runtime_storage
from stackalytics.processor import utils from stackalytics.processor import utils
LP_URI = 'https://api.launchpad.net/1.0/people/?ws.op=getByEmail&email=%s'
class TestRecordProcessor(testtools.TestCase): class TestRecordProcessor(testtools.TestCase):
def setUp(self): def setUp(self):
super(TestRecordProcessor, self).setUp() super(TestRecordProcessor, self).setUp()
@@ -89,13 +90,13 @@ class TestRecordProcessor(testtools.TestCase):
self.runtime_storage = p_storage self.runtime_storage = p_storage
self.commit_processor = record_processor.RecordProcessor(p_storage) self.commit_processor = record_processor.RecordProcessor(p_storage)
self.launchpad_patch = mock.patch('launchpadlib.launchpad.Launchpad') self.read_json_from_uri_patch = mock.patch(
self.launchpad_patch.start() 'stackalytics.processor.utils.read_json_from_uri')
cfg.CONF = mock.MagicMock() self.read_json = self.read_json_from_uri_patch.start()
def tearDown(self): def tearDown(self):
super(TestRecordProcessor, self).tearDown() super(TestRecordProcessor, self).tearDown()
self.launchpad_patch.stop() self.read_json_from_uri_patch.stop()
def _generate_commits(self, email='johndoe@gmail.com', date=1999999999): def _generate_commits(self, email='johndoe@gmail.com', date=1999999999):
yield { yield {
@@ -150,11 +151,9 @@ class TestRecordProcessor(testtools.TestCase):
""" """
email = 'johndoe@nec.co.jp' email = 'johndoe@nec.co.jp'
commit_generator = self._generate_commits(email=email) commit_generator = self._generate_commits(email=email)
lp_mock = mock.MagicMock() launchpad_id = 'john_doe'
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock) self.read_json.return_value = {'name': launchpad_id,
lp_profile = mock.Mock() 'display_name': launchpad_id}
lp_profile.name = 'john_doe'
lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
user = self.user.copy() user = self.user.copy()
# tell storage to return existing user # tell storage to return existing user
self.get_users.return_value = [user] self.get_users.return_value = [user]
@@ -163,10 +162,10 @@ class TestRecordProcessor(testtools.TestCase):
self.runtime_storage.set_by_key.assert_called_once_with('users', self.runtime_storage.set_by_key.assert_called_once_with('users',
mock.ANY) mock.ANY)
lp_mock.people.getByEmail.assert_called_once_with(email=email) self.read_json.assert_called_once_with(LP_URI % email)
self.assertIn(email, user['emails']) self.assertIn(email, user['emails'])
self.assertEquals('NEC', commit['company_name']) self.assertEquals('NEC', commit['company_name'])
self.assertEquals('john_doe', commit['launchpad_id']) self.assertEquals(launchpad_id, commit['launchpad_id'])
def test_update_commit_existing_user_new_email_unknown_company(self): def test_update_commit_existing_user_new_email_unknown_company(self):
""" """
@@ -175,11 +174,9 @@ class TestRecordProcessor(testtools.TestCase):
""" """
email = 'johndoe@yahoo.com' email = 'johndoe@yahoo.com'
commit_generator = self._generate_commits(email=email) commit_generator = self._generate_commits(email=email)
lp_mock = mock.MagicMock() launchpad_id = 'john_doe'
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock) self.read_json.return_value = {'name': launchpad_id,
lp_profile = mock.Mock() 'display_name': launchpad_id}
lp_profile.name = 'john_doe'
lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
user = self.user.copy() user = self.user.copy()
# tell storage to return existing user # tell storage to return existing user
self.get_users.return_value = [user] self.get_users.return_value = [user]
@@ -188,10 +185,10 @@ class TestRecordProcessor(testtools.TestCase):
self.runtime_storage.set_by_key.assert_called_once_with('users', self.runtime_storage.set_by_key.assert_called_once_with('users',
mock.ANY) mock.ANY)
lp_mock.people.getByEmail.assert_called_once_with(email=email) self.read_json.assert_called_once_with(LP_URI % email)
self.assertIn(email, user['emails']) self.assertIn(email, user['emails'])
self.assertEquals('SuperCompany', commit['company_name']) self.assertEquals('SuperCompany', commit['company_name'])
self.assertEquals('john_doe', commit['launchpad_id']) self.assertEquals(launchpad_id, commit['launchpad_id'])
def test_update_commit_new_user(self): def test_update_commit_new_user(self):
""" """
@@ -200,19 +197,16 @@ class TestRecordProcessor(testtools.TestCase):
""" """
email = 'smith@nec.com' email = 'smith@nec.com'
commit_generator = self._generate_commits(email=email) commit_generator = self._generate_commits(email=email)
lp_mock = mock.MagicMock() launchpad_id = 'smith'
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock) self.read_json.return_value = {'name': launchpad_id,
lp_profile = mock.Mock() 'display_name': 'Smith'}
lp_profile.name = 'smith'
lp_profile.display_name = 'Smith'
lp_mock.people.getByEmail = mock.Mock(return_value=lp_profile)
self.get_users.return_value = [] self.get_users.return_value = []
commit = list(self.commit_processor.process(commit_generator))[0] commit = list(self.commit_processor.process(commit_generator))[0]
lp_mock.people.getByEmail.assert_called_once_with(email=email) self.read_json.assert_called_once_with(LP_URI % email)
self.assertEquals('NEC', commit['company_name']) self.assertEquals('NEC', commit['company_name'])
self.assertEquals('smith', commit['launchpad_id']) self.assertEquals(launchpad_id, commit['launchpad_id'])
def test_update_commit_new_user_unknown_to_lb(self): def test_update_commit_new_user_unknown_to_lb(self):
""" """
@@ -221,32 +215,12 @@ class TestRecordProcessor(testtools.TestCase):
""" """
email = 'inkognito@avs.com' email = 'inkognito@avs.com'
commit_generator = self._generate_commits(email=email) commit_generator = self._generate_commits(email=email)
lp_mock = mock.MagicMock() self.read_json.return_value = None
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
lp_mock.people.getByEmail = mock.Mock(return_value=None)
self.get_users.return_value = [] self.get_users.return_value = []
commit = list(self.commit_processor.process(commit_generator))[0] commit = list(self.commit_processor.process(commit_generator))[0]
lp_mock.people.getByEmail.assert_called_once_with(email=email) self.read_json.assert_called_once_with(LP_URI % email)
self.assertEquals('*independent', commit['company_name'])
self.assertEquals(None, commit['launchpad_id'])
def test_update_commit_new_user_lb_raises_error(self):
"""
LP raises error during getting user info
"""
email = 'smith@avs.com'
commit_generator = self._generate_commits(email=email)
lp_mock = mock.MagicMock()
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
lp_mock.people.getByEmail = mock.Mock(return_value=None,
side_effect=Exception)
self.get_users.return_value = []
commit = list(self.commit_processor.process(commit_generator))[0]
lp_mock.people.getByEmail.assert_called_once_with(email=email)
self.assertEquals('*independent', commit['company_name']) self.assertEquals('*independent', commit['company_name'])
self.assertEquals(None, commit['launchpad_id']) self.assertEquals(None, commit['launchpad_id'])
@@ -256,13 +230,11 @@ class TestRecordProcessor(testtools.TestCase):
""" """
email = 'error.root' email = 'error.root'
commit_generator = self._generate_commits(email=email) commit_generator = self._generate_commits(email=email)
lp_mock = mock.MagicMock() self.read_json.return_value = None
launchpad.Launchpad.login_anonymously = mock.Mock(return_value=lp_mock)
lp_mock.people.getByEmail = mock.Mock(return_value=None)
self.get_users.return_value = [] self.get_users.return_value = []
commit = list(self.commit_processor.process(commit_generator))[0] commit = list(self.commit_processor.process(commit_generator))[0]
self.assertEquals(0, lp_mock.people.getByEmail.called) self.assertEquals(0, self.read_json.called)
self.assertEquals('*independent', commit['company_name']) self.assertEquals('*independent', commit['company_name'])
self.assertEquals(None, commit['launchpad_id']) self.assertEquals(None, commit['launchpad_id'])