System configuration for OpenStack Infrastructure
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

who-approves.py 6.6KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157
  1. #!/usr/bin/env python
  2. # Copyright (c) 2015 OpenStack Foundation
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13. # implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. # Description: When run using OpenStack's Gerrit server, this builds
  17. # JSON and YAML representations of repos with information on the
  18. # official owning project team if any, deliverable tags, and groups
  19. # with approve rights listing the members of each along with their
  20. # Gerrit preferred E-mail addresses and usernames when available.
  21. # Rationale: It was done as a demonstration to a representative of a
  22. # foundation member company who requested a list of the "core
  23. # reviewers" for official projects, optionally broken down by
  24. # integrated vs. other. I'm attempting to show that this data is
  25. # already publicly available and can be extracted/analyzed by anyone
  26. # without needing to request it.
  27. # Use: This needs your Gerrit username passed as the command-line
  28. # parameter, found at https://review.openstack.org/#/settings/ when
  29. # authenticated in the WebUI. It also prompts for an HTTP password
  30. # which https://review.openstack.org/#/settings/http-password will
  31. # allow you to generate. The results end up in files named
  32. # approvers.json and approvers.yaml. At the time of writing, it
  33. # takes approximately 6.5 minutes to run on a well-connected machine
  34. # with 70-80ms round-trip latency to review.openstack.org.
  35. # Example:
  36. #
  37. # $ virtualenv approvers
  38. # [...]
  39. # $ ./approvers/bin/pip install pyyaml requests
  40. # [...]
  41. # $ ./approvers/bin/python tools/who-approves.py fungi
  42. # Password:
  43. # [wait for completion]
  44. # $ ./approvers/bin/python
  45. # >>> import yaml
  46. # >>>
  47. # >>> def get_approvers(repos):
  48. # ... approvers = set()
  49. # ... for repo in repos:
  50. # ... for group in repos[repo]['approvers']:
  51. # ... for approver in repos[repo]['approvers'][group]:
  52. # ... approvers.add(approver)
  53. # ... return(approvers)
  54. # ...
  55. # >>> p = yaml.safe_load(open('approvers.yaml'))
  56. # >>> print('Total repos: %s' % len(p))
  57. # Total repos: 751
  58. # >>> print('Total approvers: %s' % len(get_approvers(p)))
  59. # Total approvers: 849
  60. # >>>
  61. # >>> o = {k: v for k, v in p.iteritems() if 'team' in v}
  62. # >>> print('Repos for official teams: %s' % len(o))
  63. # Repos for official teams: 380
  64. # >>> print('OpenStack repo approvers: %s' % len(get_approvers(o)))
  65. # OpenStack repo approvers: 456
  66. # >>>
  67. # >>> i = {k: v for k, v in p.iteritems() if 'tags' in v
  68. # ... and 'release:managed' in v['tags']}
  69. # >>> print('Repos under release management: %s' % len(i))
  70. # Repos under release management: 77
  71. # >>> print('Managed release repo approvers: %s' % len(get_approvers(i)))
  72. # Managed release repo approvers: 245
  73. import getpass
  74. import json
  75. import re
  76. import sys
  77. import requests
  78. import yaml
  79. gerrit_url = 'https://review.openstack.org/'
  80. try:
  81. gerrit_auth = requests.auth.HTTPDigestAuth(sys.argv[1], getpass.getpass())
  82. except IndexError:
  83. sys.stderr.write("Usage: %s USERNAME\n" % sys.argv[0])
  84. sys.exit(1)
  85. acl_path = 'gitweb?p=%s.git;a=blob_plain;f=project.config;hb=refs/meta/config'
  86. group_path = 'a/groups/%s/members/?recursive&pp=0'
  87. projects_file = ('gitweb?p=openstack/governance.git;a=blob_plain;'
  88. 'f=reference/projects.yaml;hb=%s')
  89. ref_name = 'refs/heads/master'
  90. aprv_pattern = 'label-Workflow = .*\.\.\+1 group (.*)'
  91. projects = requests.get(gerrit_url + projects_file % ref_name)
  92. projects.encoding = 'utf-8' # Workaround for Gitweb encoding
  93. projects = yaml.safe_load(projects.text)
  94. repos_dump = json.loads(requests.get(
  95. gerrit_url + 'projects/?pp=0').text[4:])
  96. all_groups = json.loads(requests.get(gerrit_url + 'a/groups/',
  97. auth=gerrit_auth).text[4:])
  98. repos = {}
  99. aprv_groups = {}
  100. for repo in repos_dump:
  101. repos[repo.encode('utf-8')] = {'approvers': {}}
  102. acl_ini = requests.get(gerrit_url + acl_path % repo).text
  103. for aprv_group in [str(x) for x in re.findall(aprv_pattern, acl_ini)]:
  104. if aprv_group not in repos[repo]['approvers']:
  105. repos[repo]['approvers'][aprv_group] = []
  106. if aprv_group not in aprv_groups:
  107. aprv_groups[aprv_group] = []
  108. for team in projects:
  109. if 'deliverables' in projects[team]:
  110. for deli in projects[team]['deliverables']:
  111. if 'repos' in projects[team]['deliverables'][deli]:
  112. drepos = projects[team]['deliverables'][deli]['repos']
  113. for repo in drepos:
  114. if repo in repos:
  115. repos[repo]['team'] = team
  116. if 'tags' in projects[team]['deliverables'][deli]:
  117. repos[repo]['tags'] = \
  118. projects[team]['deliverables'][deli]['tags']
  119. for aprv_group in aprv_groups.keys():
  120. # It's possible for built-in metagroups in recent Gerrit releases to
  121. # appear in ACLs but not in the groups list
  122. if aprv_group in all_groups:
  123. aprv_groups[aprv_group] = json.loads(requests.get(
  124. gerrit_url + group_path % all_groups[aprv_group]['id'],
  125. auth=gerrit_auth).text[4:])
  126. else:
  127. sys.stderr.write('Ignoring nonexistent "%s" group.\n' % aprv_group)
  128. for repo in repos:
  129. for aprv_group in repos[repo]['approvers'].keys():
  130. for approver in aprv_groups[aprv_group]:
  131. if 'name' in approver:
  132. approver_details = '"%s"' % approver['name']
  133. else:
  134. approver_details = ''
  135. if 'email' in approver:
  136. if approver_details:
  137. approver_details += ' '
  138. approver_details += '<%s>' % approver['email']
  139. if 'username' in approver:
  140. if approver_details:
  141. approver_details += ' '
  142. approver_details += '(%s)' % approver['username']
  143. repos[repo]['approvers'][aprv_group].append(
  144. approver_details.encode('utf-8'))
  145. approvers_yaml = open('approvers.yaml', 'w')
  146. yaml.dump(repos, approvers_yaml, allow_unicode=True, encoding='utf-8',
  147. default_flow_style=False)
  148. approvers_json = open('approvers.json', 'w')
  149. json.dump(repos, approvers_json, indent=2)