diff --git a/tools/atc/README b/tools/atc/README new file mode 100644 index 0000000000..4fc7cc7ac4 --- /dev/null +++ b/tools/atc/README @@ -0,0 +1,33 @@ +These are the scripts used to create the ATC lists for use in PTL +elections and Summit invitations. + +0) Write a patch to email-stats.py so steps 1-2 are not necessary. +1) Edit email-stats.py to set your gerrit username. +2) Edit email-stats.py to set the start_date and end_date. +3) Run the following queries on review.openstack.org: + + SELECT * FROM accounts + INTO OUTFILE '/tmp/accounts.csv' + FIELDS TERMINATED BY ',' + ENCLOSED BY '"' + LINES TERMINATED BY '\n'; + + SELECT * FROM account_external_ids + INTO OUTFILE '/tmp/emails.csv' + FIELDS TERMINATED BY ',' + ENCLOSED BY '"' + LINES TERMINATED BY '\n'; + +4) Copy those files to this directory. +5) Run: + + mkdir out + ./email-stats.sh + DATE=`date --iso` + mkdir $DATE + cat out/*.csv | sort | uniq > $DATE/all.csv + mv out/* $DATE/ + +6) You can use diff.py to get the new ATCs since the previous run: + + ./diff.py $OLD-DATE/all.csv $DATE/all.csv $DATE/new.csv diff --git a/tools/atc/diff.py b/tools/atc/diff.py new file mode 100755 index 0000000000..7949285d50 --- /dev/null +++ b/tools/atc/diff.py @@ -0,0 +1,16 @@ +#!/usr/bin/python + +import csv +import sys + +old = {} +new = {} + +for row in csv.reader(open(sys.argv[1])): + old[row[0]] = row + +writer = csv.writer(open(sys.argv[3], 'w')) +for row in csv.reader(open(sys.argv[2])): + if row[0] not in old: + writer.writerow(row) + diff --git a/tools/atc/email-stats.py b/tools/atc/email-stats.py new file mode 100755 index 0000000000..f332e2b01b --- /dev/null +++ b/tools/atc/email-stats.py @@ -0,0 +1,141 @@ +#!/usr/bin/python + +# Copyright (C) 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Soren Hansen wrote the original version of this script. +# James Blair hacked it up to include email addresses from gerrit. + +import calendar +import datetime +import json +import optparse +import paramiko +from pprint import pprint +import sys +import csv +import re + +MAILTO_RE = re.compile('mailto:(.*)') +USERNAME_RE = re.compile('username:(.*)') +accounts = {} + +class Account(object): + def __init__(self, num): + self.num = num + self.full_name = '' + self.emails = [] + self.username = None + +def get_account(num): + a = accounts.get(num) + if not a: + a = Account(num) + accounts[num] = a + return a + +for row in csv.reader(open('emails.csv')): + num, email, pw, external = row + num = int(num) + a = get_account(num) + if email and email != '\\N' and email not in a.emails: + a.emails.append(email) + m = MAILTO_RE.match(external) + if m: + if m.group(1) not in a.emails: + a.emails.append(m.group(1)) + m = USERNAME_RE.match(external) + if m: + if a.username: + print a.num + print a.username + raise Exception("Already a username") + a.username = m.group(1) + +for row in csv.reader(open('accounts.csv')): + num = int(row[-1]) + name = row[1] + a = get_account(num) + a.full_name = name + +username_accounts = {} +for a in accounts.values(): + username_accounts[a.username] = a + +atcs = [] + +optparser = optparse.OptionParser() +optparser.add_option('-p', '--project', default='nova', help='Project to generate stats for') +optparser.add_option('-o', '--output', default='out.csv', help='Output file') +options, args = optparser.parse_args() + +QUERY = "project:%s status:merged" % options.project + +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +client.load_system_host_keys() +client.connect('review.openstack.org', port=29418, key_filename='/home/corvus/.ssh/id_rsa', username='CHANGME') +stdin, stdout, stderr = client.exec_command('gerrit query %s --all-approvals --format JSON' % + QUERY) +changes = [] + +done = False +last_sortkey = '' +tz = datetime.tzinfo +start_date = datetime.datetime(2012, 9, 27, 0, 0, 0) +end_date = datetime.datetime(2013, 7, 30, 0, 0, 0) + +count = 0 +earliest = datetime.datetime.now() +while not done: + for l in stdout: + data = json.loads(l) + if 'rowCount' in data: + if data['rowCount'] < 500: + done = True + continue + count += 1 + last_sortkey = data['sortKey'] + if 'owner' not in data: + continue + if 'username' not in data['owner']: + continue + account = username_accounts[data['owner']['username']] + approved = False + for ps in data['patchSets']: + if 'approvals' not in ps: + continue + for aprv in ps['approvals']: + if aprv['type'] != 'SUBM': + continue + ts = datetime.datetime.fromtimestamp(aprv['grantedOn']) + if ts < start_date or ts > end_date: + continue + approved = True + if ts < earliest: + earliest = ts + if approved and account not in atcs: + atcs.append(account) + if not done: + stdin, stdout, stderr = client.exec_command('gerrit query %s resume_sortkey:%s --all-approvals --format JSON' % (QUERY, last_sortkey)) + +print 'project: %s' % options.project +print 'examined %s changes' % count +print 'earliest timestamp: %s' % earliest +writer = csv.writer(open(options.output, 'w')) +for a in atcs: + writer.writerow([a.username, a.full_name] + a.emails) +print diff --git a/tools/atc/email-stats.sh b/tools/atc/email-stats.sh new file mode 100755 index 0000000000..1fafb6848d --- /dev/null +++ b/tools/atc/email-stats.sh @@ -0,0 +1,81 @@ +#!/bin/sh + +# Integrated projects +python email-stats.py -p openstack/nova -o out/nova.csv +python email-stats.py -p openstack/swift -o out/swift.csv +python email-stats.py -p openstack/glance -o out/glance.csv +python email-stats.py -p openstack/keystone -o out/keystone.csv +python email-stats.py -p openstack/horizon -o out/horizon.csv +python email-stats.py -p openstack/neutron -o out/neutron.csv +python email-stats.py -p openstack/cinder -o out/cinder.csv +python email-stats.py -p openstack/ceilometer -o out/ceilometer.csv +python email-stats.py -p openstack/heat -o out/heat.csv + +# Library projects +python email-stats.py -p openstack/oslo-incubator -o out/olso-incubator.csv +python email-stats.py -p openstack/oslo.config -o out/olso.config.csv +python email-stats.py -p openstack/python-novaclient -o out/python-novaclient.csv +python email-stats.py -p openstack/python-swiftclient -o out/python-swiftclient.csv +python email-stats.py -p openstack/python-glanceclient -o out/python-glanceclient.csv +python email-stats.py -p openstack/python-keystoneclient -o out/python-keystoneclient.csv +python email-stats.py -p openstack/python-neutronclient -o out/python-neutronclient.csv +python email-stats.py -p openstack/python-cinderclient -o out/python-cinderclient.csv +python email-stats.py -p openstack/python-ceilometerclient -o out/python-ceilometerclient.csv +python email-stats.py -p openstack/python-heatclient -o out/python-heatclient.csv +python email-stats.py -p openstack/heat-cfntools -o out/heat-cfntools.csv +python email-stats.py -p openstack/heat-templates -o out/heat-templates.csv + +# Incubated projects +python email-stats.py -p openstack/trove -o out/trove.csv +python email-stats.py -p openstack/trove-integration -o out/trove-integration.csv +python email-stats.py -p openstack/python-troveclient -o out/python-troveclient.csv +python email-stats.py -p openstack/ironic -o out/ironic.csv +python email-stats.py -p openstack/python-ironicclient -o out/python-ironicclient.csv + +# Gating projects +python email-stats.py -p openstack-dev/devstack -o out/devstack.csv +python email-stats.py -p openstack-dev/grenade -o out/grenade.csv +python email-stats.py -p openstack-dev/hacking -o out/hacking.csv +python email-stats.py -p openstack-dev/pbr -o out/pbr.csv +python email-stats.py -p openstack/tempest -o out/tempest.csv +python email-stats.py -p openstack-dev/openstack-nose -o out/openstack-nose.csv +python email-stats.py -p openstack/requirements -o out/requirements.csv + +# Supporting projects +python email-stats.py -p openstack/compute-api -o out/compute-api.csv +python email-stats.py -p openstack/identity-api -o out/identity-api.csv +python email-stats.py -p openstack/image-api -o out/image-api.csv +python email-stats.py -p openstack/netconn-api -o out/netconn-api.csv +python email-stats.py -p openstack/object-api -o out/object-api.csv +python email-stats.py -p openstack/volume-api -o out/volume-api.csv +python email-stats.py -p openstack/openstack-manuals -o out/openstack-manuals.csv +python email-stats.py -p openstack/api-site -o out/api-site.csv +python email-stats.py -p openstack/operations-guide -o out/operations-guide.csv + +# Infrastructure projects +python email-stats.py -p openstack-infra/askbot-theme -o out/askbot-theme.csv +python email-stats.py -p openstack-infra/config -o out/config.csv +python email-stats.py -p openstack-infra/devstack-gate -o out/devstack-gate.csv +python email-stats.py -p openstack-infra/gear -o out/gear.csv +python email-stats.py -p openstack-infra/gearman-plugin -o out/gearman-plugin.csv +python email-stats.py -p openstack-infra/gerrit -o out/gerrit.csv +python email-stats.py -p openstack-infra/gerritbot -o out/gerritbot.csv +python email-stats.py -p openstack-infra/gerritlib -o out/gerritlib.csv +python email-stats.py -p openstack-infra/git-review -o out/git-review.csv +python email-stats.py -p openstack-infra/gitdm -o out/gitdm.csv +python email-stats.py -p openstack-infra/jeepyb -o out/jeepyb.csv +python email-stats.py -p openstack-infra/jenkins-job-builder -o out/jenkins-job-builder.csv +python email-stats.py -p openstack-infra/lodgeit -o out/lodgeit.csv +python email-stats.py -p openstack-infra/meetbot -o out/meetbot.csv +python email-stats.py -p openstack-infra/nose-html-output -o out/nose-html-output.csv +python email-stats.py -p openstack-infra/odsreg -o out/odsreg.csv +python email-stats.py -p openstack-infra/publications -o out/publications.csv +python email-stats.py -p openstack-infra/puppet-apparmor -o out/puppet-apparmor.csv +python email-stats.py -p openstack-infra/puppet-dashboard -o out/puppet-dashboard.csv +python email-stats.py -p openstack-infra/puppet-vcsrepo -o out/puppet-vcsrepo.csv +python email-stats.py -p openstack-infra/pypi-mirror -o out/pypi-mirror.csv +python email-stats.py -p openstack-infra/releasestatus -o out/releasestatus.csv +python email-stats.py -p openstack-infra/reviewday -o out/reviewday.csv +python email-stats.py -p openstack-infra/statusbot -o out/statusbot.csv +python email-stats.py -p openstack-infra/zmq-event-publisher -o out/zmq-event-publisher.csv +python email-stats.py -p openstack-infra/zuul -o out/zuul.csv