diff --git a/tools/owners.py b/tools/owners.py new file mode 100644 index 0000000000..c1603142e1 --- /dev/null +++ b/tools/owners.py @@ -0,0 +1,641 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +# Description: When run using OpenStack's Gerrit server, this builds +# YAML representations of aggregate change owner details and change +# counts for each governance project-team, as well as a combined set +# for all teams. + +# Rationale: The OpenStack Technical Committee and Project Team Lead +# elections need electorate rolls taken from "Active Technical +# Contributors" to any repos under official project-teams over a +# particular timeframe. Similarly, the OpenStack Foundation gives +# summit registration discount codes to contributors meeting similar +# criteria. The Gerrit REST API provides access to all the data +# necessary to identify these individuals. + +# Use: The results end up in files named for each +# official governance project-team (or "all") ending with a .yaml +# extension. At the time of writing, it takes approximately 30 +# minutes to run on a well-connected machine with 70-80ms round-trip +# latency to review.openstack.org. + +# An example for generating the March 2016 technical election rolls: +# +# $ virtualenv venv +# [...] +# $ ./venv/bin/pip install pyyaml requests +# [...] +# $ ./venv/bin/python tools/owners.py -a 2015-03-04 \ +# -b 2016-03-04 -i 11131 -o owners -r march-2016-elections +# MISSING: ansible-build-image +# MERGING DUPLICATE ACCOUNT: 8074 into 2467 +# [...blah, blah, blah...wait for completion...] +# +# TODO(fungi): Add a pass which will correctly generate the +# stable_branch_maintenance.* files. In the meantime, to properly +# generate the SBM PTL electorate, run a second time with a +# different -o of sbm, adding the -n and -s options, and then copy +# the full electorate over like: +# +# $ ./venv/bin/python tools/owners.py -a 2015-03-04 \ +# -b 2016-03-04 -i 11131 -o sbm -r march-2016-elections \ +# -n -s 'branch:^stable/.*' +# [...wait for completion again...] +# $ cp sbm/_electorate.txt owners/stable_branch_maintenance.txt +# $ cp sbm/_all_owners.yaml owners/stable_branch_maintenance.yaml +# +# Once complete, make a compressed tarball of the owners directory +# and send it attached to a PGP/MIME signed message to the appointed +# election officials. The various *.txt files are lists of the +# preferred addresses of all valid voters for the various PTL +# elections (whose team names correspond to the file names), +# suitable for passing directly to CIVS. The similarly named *.yaml +# files are detailed structured data about the same sets of voters, +# for use in validating the address lists. The _electorate.txt file +# is the equivalent address list for the TC election voters, and its +# corresponding structured data is in _all_owners.yaml. + +# You can also do interesting analysis on _all_owners.yaml, for +# example: +# +# $ ./venv/bin/python +# >>> import yaml +# >>> +# >>> o = yaml.load(open('owners/_all_owners.yaml')) +# >>> for c in range(5): +# ... print('Owners of at least %s changes: %s' % ( +# ... c+1, +# ... len({k: v for k, v in o.iteritems() if v['count'] > c}))) +# ... +# Owners of at least 1 changes: 3239 +# Owners of at least 2 changes: 2352 +# Owners of at least 3 changes: 1924 +# Owners of at least 4 changes: 1682 +# Owners of at least 5 changes: 1504 + + +from __future__ import print_function +import argparse +import csv +import datetime +import json +import os +import string +import sys + +import requests +import yaml + + +def dumper(data, stream): + """Convenience wrapper to consistently set YAML formatting""" + return yaml.safe_dump(data, allow_unicode=True, default_flow_style=False, + encoding='utf-8', stream=stream) + + +def normalize_email(email): + """Lower-case the domain part of E-mail addresses to better spot + duplicate entries, since the domain part is case-insensitive + courtesy of DNS while the local part is not necessarily""" + local, domain = email.split('@') + domain = domain.lower() + return '%s@%s' % (local, domain) + + +def normalize_project(project): + """Replace spaces and hyphens with underscores in project teams + and then lower-case them, for more convenient filenames""" + return project.translate(string.maketrans(' -', '__')).lower() + + +def date_merged(change, after=None, before=None): + """Determine the date and time a specific change merged""" + + date = change['submitted'] + + if not date: + # Something's terribly wrong with any changes matching this now + print( + 'SKIPPING DATELESS MERGE: change %s for account %s' + % (change['_number'], change['owner']['_account_id']), + file=sys.stderr) + return None + + # Strip superfluous subsecond values as Gerrit always just + # reports .000000000 for them anyway + date = date.split('.')[0] + + # Pass back an invalid result if it falls after the requested + # cutoff + if before and date >= before: + return None + + # Sanity check for completeness, but since "after" is also used + # in the Gerrit query this shouldn't ever actually be reached + if after and date < after: + return None + + return date + + +def query_gerrit(query): + """A requests wrapper to querying the Gerrit REST API""" + # TODO(fungi): this could be switched to call pygerrit instead + + # The base URL to Gerrit, for accessing both GitWeb and the REST + # API + GERRIT_API_URL = 'https://review.openstack.org/' + + # Retry http/https queries + retry = requests.Session() + retry.mount("http://", requests.adapters.HTTPAdapter(max_retries=3)) + retry.mount("https://", requests.adapters.HTTPAdapter(max_retries=3)) + raw = retry.get(GERRIT_API_URL + query, + headers={'Accept': 'application/json'}) + + # Trap decoding failures and provide more detailed errors + try: + decoded = json.loads(raw.text[4:]) + except: + print('\nrequest returned %s error to query:\n\n %s\n' + '\nwith detail:\n\n %s\n' % (raw, query, raw.text), + file=sys.stderr) + raise + return decoded + + +def main(): + """The giant pile of spaghetti which does everything else""" + + # Record the start time for use later + start = datetime.datetime.utcnow() + + # TODO(fungi): this could be trivially extracted to a separate + # function + parser = argparse.ArgumentParser( + description="When run using OpenStack's Gerrit server, this builds " + "YAML representations of aggregate change owner details and change " + "counts for each governance project-team, as well as a combined set " + "for all teams. Before and after dates/times should be supplied in " + "formats Gerrit accepts: https://review.openstack.org/Documentation/" + "user-search.html#search-operators") + parser.add_argument("-a", "--after", help="Start date for matching merges") + parser.add_argument("-b", "--before", help="End date for matching merges") + parser.add_argument("-c", "--config", help="Path to script configuration") + parser.add_argument("-i", "--ignore", help="Account Id numbers to skip", + action='append') + parser.add_argument("-n", "--no-extra-atcs", help='Omit "extra ATCs"', + dest='no_extra_atcs', action='store_true') + parser.add_argument("-o", "--outdir", help="Create an output directory") + parser.add_argument("-r", "--ref", help="Specify a Governance refname") + parser.add_argument("-s", "--sieve", help="Add Gerrit query parameters") + options = parser.parse_args() + + # If we're supplied a configuration file, use it + if options.config: + config = yaml.safe_load(open(options.config)) + # Otherwise, use nothing + else: + config = {} + + # Start of the match timeframe for change merges + if options.after: + after = options.after + elif 'after' in config: + after = config['after'] + else: + after = None + + # End of the match timeframe for change merges + if options.before: + before = options.before + elif 'before' in config: + before = config['before'] + else: + before = None + + # Owner Ids for whom to ignore changes + if options.ignore: + ignore = [int(i) for i in options.ignore] + elif 'ignore' in config: + ignore = config['ignore'] + else: + ignore = [] + + # Whether to omit "extra ATCs" + if options.no_extra_atcs: + no_extra_atcs = options.no_extra_atcs + elif 'no-extra-atcs' in config: + no_extra_atcs = config['no-extra-atcs'] + else: + no_extra_atcs = False + + # Output file directory + if options.outdir: + outdir = options.outdir + elif 'outdir' in config: + outdir = config['outdir'] + else: + outdir = '.' + if not os.path.isdir(outdir): + os.makedirs(outdir) + + # Governance Git repository ref object for reference lists + if options.ref: + ref = options.ref + elif 'ref' in config: + ref = config['ref'] + else: + ref = 'refs/heads/master' + + # Gerrit change query additions + if options.sieve: + sieve = options.sieve + elif 'sieve' in config: + sieve = config['sieve'] + else: + sieve = None + + # Path to the governance projects list, needs a Git refname as a + # parameter + PROJECTS_LIST = ('https://review.openstack.org/' + 'gitweb?p=openstack/governance.git;a=blob_plain;' + 'f=reference/projects.yaml;hb=%s') + + # The query identifying relevant changes + match = 'status:merged' + if after: + match = '%s+after:"%s"' % (match, after) + if sieve: + match = '%s+%s' % (match, sieve) + + # The set of projects from the governance repo + # TODO(fungi): make this a configurable option so that you can + # for example supply a custom project list for running elections + # in unofficial teams + gov_projects = requests.get(PROJECTS_LIST % ref) + gov_projects.encoding = 'utf-8' # Workaround for Gitweb encoding + gov_projects = yaml.safe_load(gov_projects.text) + + # A mapping of short (no prefix) to full repo names existing in + # Gerrit, used to handle repos which have a different namespace + # in governance during transitions and also to filter out repos + # listed in governance which don't actually exist + ger_repos = dict( + [(x.split('/')[-1], x) for x in query_gerrit('projects/')]) + + # This will be populated with change owners mapped to the + # project-teams maintaining their respective Git repositories + projects = {} + + # This will be populated with all change owners and their + # account details + owners = {} + + # This will be populated with discovered duplicate owners + duplicates = {} + + # This will be populated with all individual E-mail addresses of + # change owners, to facilitate finding and merging duplicate + # accounts + all_emails = {} + + # Iterate over all governance project-teams only at filename + # generation time + for project in gov_projects: + # This will be populated with change owner Ids and counts + projects[project] = {} + + # Governance project-teams have one or more deliverables + for deliverable in gov_projects[project]['deliverables']: + # Each deliverable can have multiple repos + repos = gov_projects[project]['deliverables'][deliverable]['repos'] + + # Operate on repo short-names (no namespace) to avoid + # potential namespace mismatches between governance + # and Gerrit + for repo in [r.split('/')[-1] for r in repos]: + # Only process repos which actually exist in Gerrit, + # otherwise spew a warning if skipping + if repo not in ger_repos: + print('MISSING: %s' % repo, file=sys.stderr) + else: + # Query for an arbitrary change set and get + # detailed account information about the most + # recent patchset, paginating at 100 changes + offset = 0 + changes = [] + while offset >= 0: + changes += query_gerrit( + 'changes/?q=project:%s+%s&n=100&start=%s' + '&o=CURRENT_COMMIT&o=CURRENT_REVISION' + '&o=DETAILED_ACCOUNTS' + % (ger_repos[repo], match, offset)) + if changes and changes[-1].get('_more_changes', False): + offset += 100 + else: + offset = -1 + + # Iterate over each matched change in the repo + for change in changes: + # Get the merge date and skip if it's + # outside any requested date range + merged = date_merged(change, after, before) + if not merged: + continue + + # We index owners by their unique Gerrit + # account Id numbers + owner = change['owner']['_account_id'] + + # If this owner is in the blacklist of Ids + # to skip, then move on to the next change + if owner in ignore: + continue + + # Seen this owner already? + new_owner = owner + new = False + if owner in duplicates: + owner = duplicates[owner] + elif owner not in owners: + new = True + + # For new additions, initialize this as + # their first and record specific account + # details + if new: + # Get the set of all E-mail addresses + # Gerrit knows for this owner's account + emails = query_gerrit( + 'accounts/%s/emails' + % change['owner']['_account_id']) + + # Find duplicate addresses and merge + # accounts when that happens + for email in emails: + address = normalize_email(email['email']) + if address in all_emails: + owner = all_emails[address] + duplicates[new_owner] = owner + print( + 'MERGING DUPLICATE ACCOUNT: %s into %s' + % (new_owner, owner), file=sys.stderr) + break + + # For newly found non-duplicate owners, + # initialize the global change count, + # newest/oldest merged dates, and an empty + # list where extra E-mail addresses can be + # added; also track their full name and + # Gerrit username + if new and owner == new_owner: + # TODO(fungi): this is a prime candidate + # to become a struct, or maybe a class + owners[owner] = { + 'count': 1, + 'extra': [], + 'name': change['owner'].get('name'), + 'newest': merged, + 'oldest': merged, + 'username': change['owner'].get('username'), + } + + # If we've seen this owner on another change + # in any repo then just iterate their global + # change counter and update newest/oldest + # dates + else: + owners[owner]['count'] += 1 + if merged > owners[owner]['newest']: + owners[owner]['newest'] = merged + elif merged < owners[owner]['oldest']: + owners[owner]['oldest'] = merged + + # We only want to add addresses if this is a + # new owner or a new duplicate + if new: + # Iterate over each E-mail address + for email in emails: + # Normalize the address before + # performing any matching since + # Gerrit doesn't do a great job of + # this on its own + address = normalize_email(email['email']) + + # Track this in the full list of all + # known E-mail addresses + all_emails[address] = owner + + # Whether Gerrit considers this the + # preferred E-mail address + preferred = email.get('preferred', False) + + # Store the preferred E-mail address + # under its own key since it has a + # special status, but only if this + # is not a duplicate account + if preferred and owner == new_owner: + owners[owner]['preferred'] = address + + # If this was already added to + # the extras list due to an + # additional pre-normalized + # copy, remove it there + if address in owners[owner]['extra']: + owners[owner]['extra'].remove(address) + + # Store a list of non-preferred + # addresses, deduplicating them in + # case they match post-normalization + # and treating duplicate preferred + # addresses as # non-preferred + else: + if ((address not in owners[owner]['extra']) + and (address != owners[owner].get( + 'preferred', ''))): + owners[owner]['extra'].append(address) + + # If we've seen this owner on another change + # in a repo under this project-team then + # just iterate their team change counter and + # update newest/oldest dates + if owner in projects[project]: + projects[project][owner]['count'] += 1 + if merged > projects[project][owner]['newest']: + projects[project][owner]['newest'] = merged + elif merged < projects[project][owner]['oldest']: + projects[project][owner]['oldest'] = merged + + # ...otherwise initialize this as their + # first + else: + # TODO(fungi): another potential struct + projects[project][owner] = { + 'count': 1, + 'newest': merged, + 'oldest': merged, + } + + # The negative counter will be used as a makeshift account Id + # for non-code contributors; those with owned changes use their + # Gerrit account Id instead + counter = 1 + + # Use the before time as the only contribution time for non-code + # contributors, falling back on the script start time if before + # was not specified + if before: + if len(before) == 10: + stamp = before + ' 00:00:00' + else: + stamp = before + else: + stamp = start.isoformat(sep=' ').split('.')[0] + + # Iterate over all extra-atcs entries + if not no_extra_atcs: + for project in gov_projects: + for extra_atc in gov_projects[project].get('extra-atcs', []): + name = extra_atc['name'] + email = extra_atc['email'] + address = normalize_email(email) + if address in all_emails: + owner = all_emails[address] + else: + owner = -counter + all_emails[address] = owner + owners[owner] = { + 'count': -1, + 'extra': [], + 'name': name, + 'newest': stamp, + 'oldest': stamp, + 'preferred': address, + 'username': '_non_code_contributor', + } + if owner not in projects[project]: + projects[project][owner] = { + 'count': -1, + 'newest': stamp, + 'oldest': stamp, + } + counter += 1 + + # This will hold an address list for TC electorate rolls + electorate = [] + + # A table of owners for summit invites + invites = [] + + # A fresh pass through the owners to build some other datasets + for owner in owners: + # Sort extra E-mail address lists for ease of comparison + owners[owner]['extra'].sort() + + # Build the data used for an invite + invite = [] + if 'username' not in owners[owner] or not owners[owner]['username']: + print( + 'SKIPPING MALFORMED OWNER: no username found for account %s' % + owner, file=sys.stderr) + continue + invite.append(owners[owner]['username']) + if 'name' not in owners[owner] or not owners[owner]['name']: + print( + 'SKIPPING MALFORMED OWNER: no fullname found for account %s' % + owner, file=sys.stderr) + continue + invite.append(owners[owner]['name'].encode('utf-8')) + if 'preferred' not in owners[owner]: + if 'extra' in owners[owner] and owners[owner]['extra']: + owners[owner]['preferred'] = owners[owner]['extra'][0] + owners[owner]['extra'] = owners[owner]['extra'][1:] + print( + 'MISSING PREFERRED EMAIL: used first extra address as ' + 'account %s preferred' % owner, file=sys.stderr) + else: + print( + 'SKIPPING MALFORMED OWNER: no preferred or extra ' + 'addresses found for account %s' % owner, file=sys.stderr) + continue + invite.append(owners[owner]['preferred']) + invite += owners[owner]['extra'] + invites.append(invite) + + # Append preferred addresses to the TC electorate + electorate.append(owners[owner]['preferred'] + '\n') + + # Write out a YAML file covering all change owners + fd = open(os.path.join(outdir, '_all_owners.yaml'), 'w') + dumper(owners, stream=fd) + fd.close() + + # Write out a YAML file covering tracked duplicate accounts + fd = open(os.path.join(outdir, '_duplicate_owners.yaml'), 'w') + dumper(duplicates, stream=fd) + fd.close() + + # Write out a team-specific electoral roll for CIVS + fd = open(os.path.join(outdir, '_electorate.txt'), 'w') + fd.writelines(electorate) + fd.close() + + # Write out a CSV file appropriate for the invite2summit tool + fd = open(os.path.join(outdir, '_invites.csv'), 'w') + csv.writer(fd).writerows(invites) + fd.close() + + # Make another pass through the projects so they can be dumped + # to our output files + for project in projects: + + # This will hold team-specific info for writing + output = {} + + # This will hold an address list for PTL electoral rolls + electorate = [] + + # Use a normalized project name for output file names + normalized_project = normalize_project(project) + + # Iterate over each change owner for the current team + for owner in projects[project]: + # Copy the global owner details into our output since + # we're going to modify some + output[owner] = dict(owners[owner]) + + # Replace the owner change count and newest/oldest + # merged dates with the team-specific value rather than + # using the count from the global set + for field in ('count', 'newest', 'oldest'): + output[owner][field] = projects[project][owner][field] + + # Append preferred addresses to the PTL electoral rolls + electorate.append(owners[owner]['preferred'] + '\n') + + # Write out a team-specific YAML file + fd = open(os.path.join(outdir, '%s.yaml' % normalized_project), 'w') + dumper(output, stream=fd) + fd.close() + + # Write out a team-specific electoral roll for CIVS + fd = open(os.path.join(outdir, '%s.txt' % normalized_project), 'w') + fd.writelines(electorate) + fd.close() + +main()