100 lines
3.7 KiB
Python
100 lines
3.7 KiB
Python
# Copyright (c) 2016 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import datetime
|
|
import time
|
|
|
|
import itertools
|
|
from oslo_log import log as logging
|
|
import requests
|
|
|
|
from stackalytics.processor import utils
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
DAY = 24 * 60 * 60
|
|
WEEK = 7 * DAY
|
|
|
|
ZANATA_URI = 'https://translate.openstack.org/rest/%s'
|
|
ZANATA_FIRST_RECORD = '2015-08-31' # must be Monday
|
|
|
|
zanata_session = requests.Session()
|
|
|
|
|
|
def _zanata_get_user_stats(zanata_user_id, start_date, end_date):
|
|
uri = ZANATA_URI % ('stats/user/%s/%s..%s' % (zanata_user_id,
|
|
start_date, end_date))
|
|
return utils.read_json_from_uri(uri, session=zanata_session)
|
|
|
|
|
|
def _timestamp_to_date(timestamp):
|
|
return datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d')
|
|
|
|
|
|
def _date_to_timestamp(d):
|
|
return int(time.mktime(
|
|
datetime.datetime.strptime(d, '%Y-%m-%d').timetuple()))
|
|
|
|
|
|
def log(runtime_storage_inst, translation_team_uri):
|
|
|
|
last_update_key = 'zanata:last_update'
|
|
last_update = int(runtime_storage_inst.get_by_key(last_update_key) or
|
|
_date_to_timestamp(ZANATA_FIRST_RECORD))
|
|
LOG.info('Last update: %d', last_update)
|
|
now = int(time.time())
|
|
|
|
LOG.info('Reading translation team from uri: %s', translation_team_uri)
|
|
translation_team = utils.read_yaml_from_uri(translation_team_uri)
|
|
|
|
if not translation_team:
|
|
LOG.warning('Translation team data is not available')
|
|
return
|
|
|
|
user_ids = set(u['zanata_id'] for u in runtime_storage_inst.get_all_users()
|
|
if 'zanata_id' in u)
|
|
user_ids |= set(itertools.chain.from_iterable(
|
|
team.get('translators', []) for team in translation_team.values()))
|
|
|
|
for user_id in user_ids:
|
|
for day in range(last_update, now, WEEK):
|
|
day_str = _timestamp_to_date(day)
|
|
end_str = _timestamp_to_date(day + WEEK - DAY)
|
|
user_stats = _zanata_get_user_stats(user_id, day_str, end_str)
|
|
if user_stats:
|
|
for user_stats_item in user_stats:
|
|
# Currently we only count translated words
|
|
if user_stats_item['savedState'] == 'Translated':
|
|
record = dict(
|
|
zanata_id=user_id,
|
|
date=_date_to_timestamp(
|
|
user_stats_item['savedDate']),
|
|
language_code=user_stats_item['localeId'],
|
|
language=user_stats_item['localeDisplayName'],
|
|
# Todo: not always consistent to the official name
|
|
module=user_stats_item['projectSlug'],
|
|
# Since Zanata does not support '/' character
|
|
# in project version names, i18n uses '-' instead
|
|
# of '/' for branch names.
|
|
branch=user_stats_item['versionSlug'].replace(
|
|
'-', '/'),
|
|
translated=user_stats_item['wordCount'],
|
|
)
|
|
yield record
|
|
last_update += (now - last_update) // WEEK * WEEK
|
|
LOG.info('New last update: %d', last_update)
|
|
runtime_storage_inst.set_by_key(last_update_key, last_update)
|