#!/usr/bin/env python # Copyright (C) 2013-2014 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # # See the License for the specific language governing permissions and # limitations under the License. # # Soren Hansen wrote the original version of this script. # James Blair hacked it up to include email addresses from gerrit. # Jeremy Stanley overhauled it for gerrit 2.8 and our governance repo. import csv import datetime import json import optparse import os import os.path import re import paramiko import requests import yaml MAILTO_RE = re.compile('mailto:(.*)') USERNAME_RE = re.compile('username:(.*)') EXTRA_ATC_RE = re.compile('^[^#][^:]*: ([^\(]*) \(([^@]*@[^\)]*)\) \[[^\[]*\]') PROJECTS_URL = ('https://git.openstack.org/cgit/openstack/governance/plain' '/reference/projects.yaml') EXTRA_ATCS_URL = ('https://git.openstack.org/cgit/openstack/governance/plain' '/reference/extra-atcs') class Account(object): def __init__(self, num): self.num = num self.full_name = '' self.emails = [] self.username = None def get_account(accounts, num): a = accounts.get(num) if not a: a = Account(num) accounts[num] = a return a def repo_stats(repo, output, begin, end, keyfile, user): accounts = {} for row in open('accounts.tab'): if not row.startswith('registered_on'): row = row.split('\t') num = int(row[13]) name = row[1] email = row[2] a = get_account(accounts, num) a.full_name = name if email and email != 'NULL': a.emails.append(email) for row in open('emails.tab'): if not row.startswith('account_id'): num, email, pw, external = row.split('\t') num = int(num) a = get_account(accounts, num) if email and email != 'NULL' and email not in a.emails: a.emails.append(email) m = MAILTO_RE.match(external) if m: if m.group(1) not in a.emails: a.emails.append(m.group(1)) m = USERNAME_RE.match(external) if m: if a.username: print a.num print a.username raise Exception("Already a username") a.username = m.group(1) username_accounts = {} for a in accounts.values(): username_accounts[a.username] = a atcs = [] QUERY = "project:%s status:merged" % repo client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.load_system_host_keys() client.connect( 'review.opendev.org', port=29418, key_filename=os.path.expanduser(keyfile), username=user) stdin, stdout, stderr = client.exec_command( 'gerrit query %s --all-approvals --format JSON' % QUERY) done = False last_sortkey = '' begin_time = datetime.datetime( int(begin[0:4]), int(begin[4:6]), int(begin[6:8]), int(begin[8:10]), int(begin[10:12]), int(begin[12:14])) end_time = datetime.datetime( int(end[0:4]), int(end[4:6]), int(end[6:8]), int(end[8:10]), int(end[10:12]), int(end[12:14])) count = 0 earliest = datetime.datetime.now() while not done: for l in stdout: data = json.loads(l) if 'rowCount' in data: if data['rowCount'] < 500: done = True continue count += 1 last_sortkey = data['sortKey'] if 'owner' not in data: continue if 'username' not in data['owner']: continue account = username_accounts[data['owner']['username']] approved = False for ps in data['patchSets']: if 'approvals' not in ps: continue for aprv in ps['approvals']: if aprv['type'] != 'SUBM': continue ts = datetime.datetime.fromtimestamp(aprv['grantedOn']) if ts < begin_time or ts > end_time: continue approved = True if ts < earliest: earliest = ts if approved and account not in atcs: atcs.append(account) if not done: stdin, stdout, stderr = client.exec_command( 'gerrit query %s resume_sortkey:%s --all-approvals' ' --format JSON' % (QUERY, last_sortkey)) print 'repo: %s' % repo print 'examined %s changes' % count print 'earliest timestamp: %s' % earliest writer = csv.writer(open(output, 'w')) for a in atcs: writer.writerow([a.username, a.full_name] + a.emails) print def get_repos(url, keyfile, user): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.load_system_host_keys() client.connect( 'review.opendev.org', port=29418, key_filename=os.path.expanduser(keyfile), username=user) stdin, stdout, stderr = client.exec_command('gerrit ls-projects') gerrit_repos = [x.strip() for x in stdout] projects_yaml = yaml.load(requests.get(url).text) repos = [] for team in projects_yaml: for deliver in projects_yaml[team]['deliverables']: for repo in projects_yaml[team]['deliverables'][deliver]['repos']: for r in gerrit_repos: if re.search('/%s$' % repo.split('/')[1], r): repos.append(r) break return repos def get_extra_atcs(url): extra_atcs = [] for line in requests.get(url).text.split('\n'): if line and not line.startswith('#'): extra_atcs.append(line) return extra_atcs def main(): now = ''.join( '%02d' % x for x in datetime.datetime.utcnow().utctimetuple()[:6]) optparser = optparse.OptionParser() optparser.add_option( '-b', '--begin', help='begin date/time (e.g. 20131017000000)') optparser.add_option( '-e', '--end', default=now, help='end date/time (default is now)') optparser.add_option( '-k', '--keyfile', default='~/.ssh/id_rsa', help='SSH key (default is ~/.ssh/id_rsa)') optparser.add_option( '-r', '--ref', default='', help='governance git ref (e.g. sept-2014-elections') optparser.add_option( '-u', '--user', default=os.environ['USER'], help='SSH username (default is $USER)') options, args = optparser.parse_args() if options.ref: projects_url = '%s?id=%s' % (PROJECTS_URL, options.ref) extra_atcs_url = '%s?id=%s' % (EXTRA_ATCS_URL, options.ref) else: projects_url = PROJECTS_URL extra_atcs_url = EXTRA_ATCS_URL for repo in get_repos(projects_url, options.keyfile, options.user): output = 'out/%s.csv' % repo.split('/')[-1] repo_stats(repo, output, options.begin, options.end, options.keyfile, options.user) writer = csv.writer(open('out/extra-atcs.csv', 'w')) for atc in get_extra_atcs(extra_atcs_url): try: writer.writerow([''] + list(EXTRA_ATC_RE.match(atc).groups())) except AttributeError: pass if __name__ == "__main__": main()