x/stackalytics
Code Issues Proposed changes

use new zanata api to query user stats

Browse Source 
now use one api to query user stats for
all projects.
so remove the project filter and version
filter.

Change-Id: I468d82f779555edd9756d401749f721a9979f46e
This commit is contained in:
Shengjing Zhu
2016-03-05 22:44:22 +08:00
parent fd40ba8c4c
commit 44aeaea892
1 changed files with 24 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
stackalytics/processor/zanata.py
View File

@@ -14,7 +14,6 @@
# limitations under the License. # limitations under the License.
import datetime import datetime
import re
import time import time
import itertools import itertools
@@ -32,36 +31,12 @@ WEEK = 7 * DAY
ZANATA_URI = 'https://translate.openstack.org/rest/%s' ZANATA_URI = 'https://translate.openstack.org/rest/%s'
ZANATA_FIRST_RECORD = '2015-08-31' # must be Monday ZANATA_FIRST_RECORD = '2015-08-31' # must be Monday
# We limit the projects and versions to reduce number of requests to Zanata API
ZANATA_VERSION_PATTERN = re.compile(r'^(master)$')
ZANATA_PROJECT_PATTERN = re.compile(r'(horizon$|.*guide|.*manual|.*doc)')
zanata_session = requests.Session() zanata_session = requests.Session()
def _zanata_get_projects(): def _zanata_get_user_stats(zanata_user_id, start_date, end_date):
uri = ZANATA_URI % 'projects' uri = ZANATA_URI % ('stats/user/%s/%s..%s' % (zanata_user_id,
LOG.debug("Reading projects from %s" % uri) start_date, end_date))
projects_data = utils.read_json_from_uri(uri, session=zanata_session)
return (p['id'] for p in projects_data
if ZANATA_PROJECT_PATTERN.match(p['id']))
def _zanata_get_project_versions(project_id):
LOG.debug("Reading iterations for project %s" % project_id)
uri = ZANATA_URI % ('projects/p/%s' % project_id)
project_data = utils.read_json_from_uri(uri, session=zanata_session)
return (it['id'] for it in project_data.get('iterations', [])
if ZANATA_VERSION_PATTERN.match(it['id']))
def _zanata_get_user_stats(project_id, iteration_id, zanata_user_id,
start_date, end_date):
uri = ZANATA_URI % ('stats/project/%s/version/%s/contributor/%s/%s..%s'
% (project_id, iteration_id, zanata_user_id,
start_date, end_date))
return utils.read_json_from_uri(uri, session=zanata_session) return utils.read_json_from_uri(uri, session=zanata_session)
@@ -89,40 +64,32 @@ def log(runtime_storage_inst, translation_team_uri):
LOG.warning('Translation team data is not available') LOG.warning('Translation team data is not available')
return return
languages = dict((k, v['language'][0])
for k, v in translation_team.items())
user_ids = set(u['zanata_id'] for u in runtime_storage_inst.get_all_users() user_ids = set(u['zanata_id'] for u in runtime_storage_inst.get_all_users()
if 'zanata_id' in u) if 'zanata_id' in u)
user_ids |= set(itertools.chain.from_iterable( user_ids |= set(itertools.chain.from_iterable(
team.get('translators', []) for team in translation_team.values())) team.get('translators', []) for team in translation_team.values()))
for project_id in _zanata_get_projects(): for user_id in user_ids:
for version in _zanata_get_project_versions(project_id): for day in range(last_update, now, WEEK):
for user_id in user_ids: day_str = _timestamp_to_date(day)
end_str = _timestamp_to_date(day + WEEK - DAY)
for day in range(last_update, now, WEEK): user_stats = _zanata_get_user_stats(user_id, day_str, end_str)
day_str = _timestamp_to_date(day) if user_stats:
end_str = _timestamp_to_date(day + WEEK - DAY) for user_stats_item in user_stats:
# Currently we only count translated words
stats = _zanata_get_user_stats( if user_stats_item['savedState'] == 'Translated':
project_id, version, user_id, day_str, end_str) record = dict(
user_stats = stats[user_id] zanata_id=user_id,
date=day,
if user_stats: language_code=user_stats_item['localeId'],
for lang, data in user_stats.items(): language=user_stats_item['localeDisplayName'],
record = dict( # Todo: not always consistent to the official name
zanata_id=user_id, module=user_stats_item['projectSlug'],
date=day, # Todo: adapt version to branch
language_code=lang, branch=user_stats_item['versionSlug'],
language=languages.get(lang) or lang, translated=user_stats_item['wordCount'],
translated=data['translated'], )
approved=data['approved'], yield record
module=project_id,
branch=version, # todo adapt version to branch
)
yield record
last_update += (now - last_update) // WEEK * WEEK last_update += (now - last_update) // WEEK * WEEK
LOG.info('New last update: %d', last_update) LOG.info('New last update: %d', last_update)
runtime_storage_inst.set_by_key(last_update_key, last_update) runtime_storage_inst.set_by_key(last_update_key, last_update)