Add fetching_user_source config option

Launchpad manages many information related to OpenStack at this
time, but the query takes much time for the processor. In addition,
stackalytics can be useful for OSS which are not managed with the
launchpad. So this patch adds a config option fetching_source so
that users can select fetching source. The default option is
'launchpad', so this patch doesn't affect at all on the default
option.

Change-Id: I4b7db745c36545e8897132cf6ccbb1c3fd6f3c07

doc/source/tools/stackalytics-processor.txt

@@ -3,6 +3,7 @@ usage: stackalytics-processor [-h] [--config-dir DIR] [--config-file PATH]
                               [--days_to_update_members DAYS_TO_UPDATE_MEMBERS]
                               [--debug] [--default-data-uri DEFAULT_DATA_URI]
                               [--driverlog-data-uri DRIVERLOG_DATA_URI]
+                              [--fetching-user-source FETCHING_USER_SOURCE]
                               [--gerrit-retry GERRIT_RETRY]
                               [--git-base-uri GIT_BASE_URI]
                               [--log-config-append PATH]
@@ -44,6 +45,8 @@ optional arguments:
                         file:///path/to/default_data.json
   --driverlog-data-uri DRIVERLOG_DATA_URI
                         URI for default data
+  --fetching-user-source FETCHING_USER_SOURCE
+                        Source for fetching user profiles
   --gerrit-retry GERRIT_RETRY
                         How many times to retry after Gerrit errors
   --git-base-uri GIT_BASE_URI

etc/stackalytics.conf

@@ -147,6 +147,10 @@
 # URI of translation team data (string value)
 #translation_team_uri = https://git.openstack.org/cgit/openstack/i18n/plain/tools/zanata/translation_team.yaml
 
+# Source for fetching user profiles (string value)
+# Allowed values: launchpad, <None>
+#fetching_user_source = launchpad
+
 # How many member profiles to look ahead after the last (integer value)
 #members_look_ahead = 250
 

stackalytics/processor/config.py

@@ -54,6 +54,9 @@ PROCESSOR_OPTS = [
                default='https://git.openstack.org/cgit/openstack/i18n/'
                        'plain/tools/zanata/translation_team.yaml',
                help='URI of translation team data'),
+    cfg.StrOpt("fetching-user-source", default='launchpad',
+               choices=['launchpad', '<None>'],
+               help="Source for fetching user profiles"),
     cfg.IntOpt('members-look-ahead', default=250,
                help='How many member profiles to look ahead after the last'),
     cfg.IntOpt('read-timeout', default=120,

stackalytics/processor/record_processor.py

@@ -19,6 +19,7 @@ import copy
 import functools
 import time
 
+from oslo_config import cfg
 from oslo_log import log as logging
 import six
 
@@ -27,6 +28,7 @@ from stackalytics.processor import user_processor
 from stackalytics.processor import utils
 
 
+CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
 
@@ -232,6 +234,9 @@ class RecordProcessor(object):
                         self.runtime_storage_inst, u)
         return merged_user
 
+    def _need_to_fetch_launchpad(self):
+        return CONF.fetching_user_source == 'launchpad'
+
     def update_user(self, record):
         email = record.get('author_email')
         user_e = user_processor.load_user(
@@ -239,8 +244,8 @@ class RecordProcessor(object):
 
         user_name = record.get('author_name')
         launchpad_id = record.get('launchpad_id')
-        if (email and (not user_e) and (not launchpad_id) and
-                (not user_e.get('launchpad_id'))):
+        if (self._need_to_fetch_launchpad() and email and (not user_e) and
+                (not launchpad_id) and (not user_e.get('launchpad_id'))):
             # query LP
             launchpad_id, lp_user_name = self._get_lp_info(email)
             if lp_user_name:
@@ -250,8 +255,8 @@ class RecordProcessor(object):
         if gerrit_id:
             user_g = user_processor.load_user(
                 self.runtime_storage_inst, gerrit_id=gerrit_id) or {}
-            if ((not user_g) and (not launchpad_id) and
-                    (not user_e.get('launchpad_id'))):
+            if (self._need_to_fetch_launchpad() and (not user_g) and
+                    (not launchpad_id) and (not user_e.get('launchpad_id'))):
                 # query LP
                 guessed_lp_id = gerrit_id
                 lp_user_name = self._get_lp_user_name(guessed_lp_id)
@@ -264,8 +269,8 @@ class RecordProcessor(object):
         if zanata_id:
             user_z = user_processor.load_user(
                 self.runtime_storage_inst, zanata_id=zanata_id) or {}
-            if ((not user_z) and (not launchpad_id) and
-                    (not user_e.get('launchpad_id'))):
+            if (self._need_to_fetch_launchpad() and (not user_z) and
+                    (not launchpad_id) and (not user_e.get('launchpad_id'))):
                 # query LP
                 guessed_lp_id = zanata_id
                 user_name = self._get_lp_user_name(guessed_lp_id)
@@ -290,7 +295,7 @@ class RecordProcessor(object):
                     [user_e, user_l, user_g, user_z, user])
             else:
                 # create new
-                if not user_name:
+                if (self._need_to_fetch_launchpad() and not user_name):
                     user_name = self._get_lp_user_name(launchpad_id)
                     if user_name:
                         user['user_name'] = user_name

stackalytics/tests/unit/test_record_processor.py

@@ -16,15 +16,19 @@
 import time
 
 import mock
+from oslo_config import cfg
 import six
 import testtools
 
+from stackalytics.processor import config
 from stackalytics.processor import record_processor
 from stackalytics.processor import runtime_storage
 from stackalytics.processor import user_processor
 from stackalytics.processor import utils
 
 
+CONF = cfg.CONF
+
 RELEASES = [
     {
         'release_name': 'prehistory',
@@ -67,6 +71,7 @@ class TestRecordProcessor(testtools.TestCase):
         self.lp_profile_by_email = (
             self.lp_profile_by_email_patch.start())
         self.lp_profile_by_email.return_value = None
+        CONF.register_opts(config.CONNECTION_OPTS + config.PROCESSOR_OPTS)
 
     def tearDown(self):
         super(TestRecordProcessor, self).tearDown()