opendev/pypi-mirror
Code Issues Proposed changes

Strip out the things we don't need from jeepyb

Browse Source 
Change-Id: I129b6d66840604898592de31df11b3e396cd6048
This commit is contained in:
Monty Taylor 2013-09-09 13:41:57 -05:00
parent 384c02d41f
commit c28dde9d68
21 changed files with 19 additions and 2058 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MANIFEST.in
View File

@ -1,4 +1,3 @@
include jeepyb/versioninfo
include AUTHORS
include ChangeLog

17
README.rst
View File

@ -1,7 +1,12 @@
===============================
Tools to Manage Gerrit Projects
===============================
====================
Partial PyPI Mirrors
====================
jeepyb is a collection of tools which make managing a gerrit easier.
Specifically, management of gerrit projects and their associated upstream
integration with things like github and launchpad.
Sometimes you want a PyPI mirror, but you don't want the whole thing. You
certainly don't want external links. What you want are the things that you
need and nothing more. What's more, you often know exactly what you need
because you already have a pip requirements.txt file containing the list of
things you expect to download from PyPI.
pypi-mirror will build a local static mirror for you based on requirements
files in git repos.

105
jeepyb/cmd/close_pull_requests.py
View File

@ -1,105 +0,0 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Github pull requests closer reads a project config file called projects.yaml
# It should look like:
# - homepage: http://openstack.org
# team-id: 153703
# has-wiki: False
# has-issues: False
# has-downloads: False
# ---
# - project: PROJECT_NAME
# options:
# - has-pull-requests
# Github authentication information is read from github.secure.config,
# which should look like:
# [github]
# username = GITHUB_USERNAME
# password = GITHUB_PASSWORD
#
# or
#
# [github]
# oauth_token = GITHUB_OAUTH_TOKEN
import ConfigParser
import github
import logging
import os
import yaml
MESSAGE = """Thank you for contributing to %(project)s!
%(project)s uses Gerrit for code review.
Please visit http://wiki.openstack.org/GerritWorkflow and follow the
instructions there to upload your change to Gerrit.
"""
def main():
logging.basicConfig(level=logging.ERROR)
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
'/etc/github/github.secure.config')
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
(defaults, config) = [config for config in
yaml.load_all(open(PROJECTS_YAML))]
if secure_config.has_option("github", "oauth_token"):
ghub = github.Github(secure_config.get("github", "oauth_token"))
else:
ghub = github.Github(secure_config.get("github", "username"),
secure_config.get("github", "password"))
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
for section in config:
project = section['project']
# Make sure we're supposed to close pull requests for this project:
if 'options' in section and 'has-pull-requests' in section['options']:
continue
# Find the project's repo
project_split = project.split('/', 1)
if len(project_split) > 1:
org = orgs_dict[project_split[0].lower()]
repo = org.get_repo(project_split[1])
else:
repo = ghub.get_user().get_repo(project)
# Close each pull request
pull_requests = repo.get_pulls("open")
for req in pull_requests:
vars = dict(project=project)
issue_data = {"url": repo.url + "/issues/" + str(req.number)}
issue = github.Issue.Issue(req._requester,
issue_data,
completed=True)
issue.create_comment(MESSAGE % vars)
req.edit(state="closed")
if __name__ == "__main__":
main()

72
jeepyb/cmd/create_cgitrepos.py
View File

@ -1,72 +0,0 @@
#! /usr/bin/env python
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# create_cgitrepos.py reads the project config file called projects.yaml
# and generates a cgitrepos configuration file which is then copied to
# the cgit server.
#
# It also creates the necessary top-level directories for each project
# organization (openstack, stackforge, etc)
import os
import subprocess
import yaml
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/cgit/projects.yaml')
CGIT_REPOS = os.environ.get('CGIT_REPOS',
'/etc/cgitrepos')
REPO_PATH = os.environ.get('REPO_PATH',
'/var/lib/git')
CGIT_USER = os.environ.get('CGIT_USER', 'cgit')
CGIT_GROUP = os.environ.get('CGIT_GROUP', 'cgit')
def main():
(defaults, config) = tuple(yaml.safe_load_all(open(PROJECTS_YAML)))
gitorgs = {}
names = set()
for entry in config:
(org, name) = entry['project'].split('/')
description = entry.get('description', name)
assert name not in names
names.add(name)
gitorgs.setdefault(org, []).append((name, description))
for org in gitorgs:
if not os.path.isdir('%s/%s' % (REPO_PATH, org)):
os.makedirs('%s/%s' % (REPO_PATH, org))
with open(CGIT_REPOS, 'w') as cgit_file:
cgit_file.write('# Autogenerated by create_cgitrepos.py\n')
for org in sorted(gitorgs):
cgit_file.write('\n')
cgit_file.write('section=%s\n' % (org))
org_dir = os.path.join(REPO_PATH, org)
projects = gitorgs[org]
projects.sort()
for (name, description) in projects:
project_repo = "%s.git" % os.path.join(org_dir, name)
cgit_file.write('\n')
cgit_file.write('repo.url=%s/%s\n' % (org, name))
cgit_file.write('repo.path=%s/\n' % (project_repo))
cgit_file.write('repo.desc=%s\n' % (description))
if not os.path.exists(project_repo):
subprocess.call(['git', 'init', '--bare', project_repo])
subprocess.call(['chown', '-R', '%s:%s'
% (CGIT_USER, CGIT_GROUP), project_repo])
if __name__ == "__main__":
main()

87
jeepyb/cmd/expire_old_reviews.py
View File

@ -1,87 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This script is designed to expire old code reviews that have not been touched
# using the following rules:
# 1. if negative comment and no activity in 1 week, expire
import argparse
import json
import logging
import paramiko
logger = logging.getLogger('expire_reviews')
logger.setLevel(logging.INFO)
def expire_patch_set(ssh, patch_id, patch_subject):
message = ('code review expired after 1 week of no activity'
' after a negative review, it can be restored using'
' the \`Restore Change\` button under the Patch Set'
' on the web interface')
command = ('gerrit review --abandon '
'--message="{message}" {patch_id}').format(
message=message,
patch_id=patch_id)
logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message)
stdin, stdout, stderr = ssh.exec_command(command)
if stdout.channel.recv_exit_status() != 0:
logger.error(stderr.read())
def main():
parser = argparse.ArgumentParser()
parser.add_argument('user', help='The gerrit admin user')
parser.add_argument('ssh_key', help='The gerrit admin SSH key file')
options = parser.parse_args()
GERRIT_USER = options.user
GERRIT_SSH_KEY = options.ssh_key
logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s'
' - %(message)s',
filename='/var/log/gerrit/expire_reviews.log')
logger.info('Starting expire reviews')
logger.info('Connecting to Gerrit')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('localhost', username=GERRIT_USER,
key_filename=GERRIT_SSH_KEY, port=29418)
# Query all reviewed with no activity for 1 week
logger.info('Searching no activity on negative review for 1 week')
stdin, stdout, stderr = ssh.exec_command(
'gerrit query --current-patch-set --all-approvals'
' --format JSON status:reviewed age:1w')
for line in stdout:
row = json.loads(line)
if 'rowCount' not in row:
# Search for negative approvals
for approval in row['currentPatchSet']['approvals']:
if approval['value'] in ('-1', '-2'):
expire_patch_set(ssh,
row['currentPatchSet']['revision'],
row['subject'])
break
logger.info('End expire review')
if __name__ == "__main__":
main()

82
jeepyb/cmd/fetch_remotes.py
View File

@ -1,82 +0,0 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Fetch remotes reads a project config file called projects.yaml
# It should look like:
# - homepage: http://openstack.org
# team-id: 153703
# has-wiki: False
# has-issues: False
# has-downloads: False
# ---
# - project: PROJECT_NAME
# options:
# - remote: https://gerrit.googlesource.com/gerrit
import logging
import os
import shlex
import subprocess
import yaml
def run_command(cmd, status=False, env={}):
cmd_list = shlex.split(str(cmd))
newenv = os.environ
newenv.update(env)
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=newenv)
(out, nothing) = p.communicate()
if status:
return (p.returncode, out.strip())
return out.strip()
def run_command_status(cmd, env={}):
return run_command(cmd, True, env)
def main():
logging.basicConfig(level=logging.ERROR)
REPO_ROOT = os.environ.get('REPO_ROOT',
'/home/gerrit2/review_site/git')
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
(defaults, config) = [config for config in
yaml.load_all(open(PROJECTS_YAML))]
for section in config:
project = section['project']
if 'remote' not in section:
continue
project_git = "%s.git" % project
os.chdir(os.path.join(REPO_ROOT, project_git))
# Make sure that the specified remote exists
remote_url = section['remote']
# We could check if it exists first, but we're ignoring output anyway
# So just try to make it, and it'll either make a new one or do nothing
run_command("git remote add -f upstream %s" % remote_url)
# Fetch new revs from it
run_command("git remote update upstream")
if __name__ == "__main__":
main()

412
jeepyb/cmd/manage_projects.py
View File

@ -1,412 +0,0 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# manage_projects.py reads a project config file called projects.yaml
# It should look like:
# - homepage: http://openstack.org
# gerrit-host: review.openstack.org
# local-git-dir: /var/lib/git
# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
# gerrit-committer: Project Creator <openstack-infra@lists.openstack.org>
# has-github: True
# has-wiki: False
# has-issues: False
# has-downloads: False
# acl-dir: /home/gerrit2/acls
# acl-base: /home/gerrit2/acls/project.config
# ---
# - project: PROJECT_NAME
# options:
# - has-wiki
# - has-issues
# - has-downloads
# - has-pull-requests
# homepage: Some homepage that isn't http://openstack.org
# description: This is a great project
# remote: https://gerrit.googlesource.com/gerrit
# upstream: git://github.com/bushy/beards.git
# acl-config: /path/to/gerrit/project.config
# acl-append:
# - /path/to/gerrit/project.config
# acl-parameters:
# project: OTHER_PROJECT_NAME
from __future__ import print_function
import ConfigParser
import logging
import os
import re
import shlex
import subprocess
import tempfile
import yaml
import gerritlib.gerrit
import github
import jeepyb.gerritdb
logging.basicConfig(level=logging.ERROR)
log = logging.getLogger("manage_projects")
def run_command(cmd, status=False, env={}):
cmd_list = shlex.split(str(cmd))
newenv = os.environ
newenv.update(env)
log.debug("Executing command: %s" % " ".join(cmd_list))
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=newenv)
(out, nothing) = p.communicate()
log.debug("Return code: %s" % p.returncode)
log.debug("Command said: %s" % out.strip())
if status:
return (p.returncode, out.strip())
return out.strip()
def run_command_status(cmd, env={}):
return run_command(cmd, True, env)
def git_command(repo_dir, sub_cmd, env={}):
git_dir = os.path.join(repo_dir, '.git')
cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
status, _ = run_command(cmd, True, env)
return status
def git_command_output(repo_dir, sub_cmd, env={}):
git_dir = os.path.join(repo_dir, '.git')
cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
status, out = run_command(cmd, True, env)
return (status, out)
def write_acl_config(project, acl_dir, acl_base, acl_append, parameters):
project_parts = os.path.split(project)
if len(project_parts) > 1:
repo_base = os.path.join(acl_dir, *project_parts[:-1])
if not os.path.exists(repo_base):
os.makedirs(repo_base)
if not os.path.isdir(repo_base):
return 1
project = project_parts[-1]
config_file = os.path.join(repo_base, "%s.config" % project)
else:
config_file = os.path.join(acl_dir, "%s.config" % project)
if 'project' not in parameters:
parameters['project'] = project
with open(config_file, 'w') as config:
if acl_base and os.path.exists(acl_base):
config.write(open(acl_base, 'r').read())
for acl_snippet in acl_append:
if not os.path.exists(acl_snippet):
acl_snippet = os.path.join(acl_dir, acl_snippet)
if not os.path.exists(acl_snippet):
continue
with open(acl_snippet, 'r') as append_content:
config.write(append_content.read() % parameters)
def fetch_config(project, remote_url, repo_path, env={}):
status = git_command(repo_path, "fetch %s +refs/meta/config:"
"refs/remotes/gerrit-meta/config" % remote_url, env)
if status != 0:
print("Failed to fetch refs/meta/config for project: %s" % project)
return False
# Because the following fails if executed more than once you should only
# run fetch_config once in each repo.
status = git_command(repo_path, "checkout -b config "
"remotes/gerrit-meta/config")
if status != 0:
print("Failed to checkout config for project: %s" % project)
return False
return True
def copy_acl_config(project, repo_path, acl_config):
if not os.path.exists(acl_config):
return False
acl_dest = os.path.join(repo_path, "project.config")
status, _ = run_command("cp %s %s" %
(acl_config, acl_dest), status=True)
if status == 0:
status = git_command(repo_path, "diff --quiet")
if status != 0:
return True
return False
def push_acl_config(project, remote_url, repo_path, gitid, env={}):
cmd = "commit -a -m'Update project config.' --author='%s'" % gitid
status = git_command(repo_path, cmd)
if status != 0:
print("Failed to commit config for project: %s" % project)
return False
status, out = git_command_output(repo_path,
"push %s HEAD:refs/meta/config" %
remote_url, env)
if status != 0:
print("Failed to push config for project: %s" % project)
print(out)
return False
return True
def _get_group_uuid(gerrit, group):
cursor = jeepyb.gerritdb.connect().cursor()
query = "SELECT group_uuid FROM account_groups WHERE name = %s"
cursor.execute(query, group)
data = cursor.fetchone()
if data:
return data[0]
return None
def get_group_uuid(gerrit, group):
uuid = _get_group_uuid(gerrit, group)
if uuid:
return uuid
gerrit.createGroup(group)
uuid = _get_group_uuid(gerrit, group)
if uuid:
return uuid
return None
def create_groups_file(project, gerrit, repo_path):
acl_config = os.path.join(repo_path, "project.config")
group_file = os.path.join(repo_path, "groups")
uuids = {}
for line in open(acl_config, 'r'):
r = re.match(r'^\s+.*group\s+(.*)$', line)
if r:
group = r.group(1)
if group in uuids.keys():
continue
uuid = get_group_uuid(gerrit, group)
if uuid:
uuids[group] = uuid
else:
return False
if uuids:
with open(group_file, 'w') as fp:
for group, uuid in uuids.items():
fp.write("%s\t%s\n" % (uuid, group))
status = git_command(repo_path, "add groups")
if status != 0:
print("Failed to add groups file for project: %s" % project)
return False
return True
def make_ssh_wrapper(gerrit_user, gerrit_key):
(fd, name) = tempfile.mkstemp(text=True)
os.write(fd, '#!/bin/bash\n')
os.write(fd,
'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
(gerrit_key, gerrit_user))
os.close(fd)
os.chmod(name, 0o755)
return dict(GIT_SSH=name)
def create_github_project(defaults, options, project, description, homepage):
default_has_issues = defaults.get('has-issues', False)
default_has_downloads = defaults.get('has-downloads', False)
default_has_wiki = defaults.get('has-wiki', False)
has_issues = 'has-issues' in options or default_has_issues
has_downloads = 'has-downloads' in options or default_has_downloads
has_wiki = 'has-wiki' in options or default_has_wiki
GITHUB_SECURE_CONFIG = defaults.get(
'github-config',
'/etc/github/github-projects.secure.config')
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
# Project creation doesn't work via oauth
ghub = github.Github(secure_config.get("github", "username"),
secure_config.get("github", "password"))
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
# Find the project's repo
project_split = project.split('/', 1)
org_name = project_split[0]
if len(project_split) > 1:
repo_name = project_split[1]
else:
repo_name = project
try:
org = orgs_dict[org_name.lower()]
except KeyError:
# We do not have control of this github org ignore the project.
return
try:
repo = org.get_repo(repo_name)
except github.GithubException:
repo = org.create_repo(repo_name,
homepage=homepage,
has_issues=has_issues,
has_downloads=has_downloads,
has_wiki=has_wiki)
if description:
repo.edit(repo_name, description=description)
if homepage:
repo.edit(repo_name, homepage=homepage)
repo.edit(repo_name, has_issues=has_issues,
has_downloads=has_downloads,
has_wiki=has_wiki)
if 'gerrit' not in [team.name for team in repo.get_teams()]:
teams = org.get_teams()
teams_dict = dict(zip([t.name.lower() for t in teams], teams))
teams_dict['gerrit'].add_to_repos(repo)
def main():
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
defaults = configs[0][0]
default_has_github = defaults.get('has-github', True)
LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
ACL_DIR = defaults.get('acl-dir')
GERRIT_HOST = defaults.get('gerrit-host')
GERRIT_PORT = int(defaults.get('gerrit-port', '29418'))
GERRIT_USER = defaults.get('gerrit-user')
GERRIT_KEY = defaults.get('gerrit-key')
GERRIT_GITID = defaults.get('gerrit-committer')
GERRIT_SYSTEM_USER = defaults.get('gerrit-system-user', 'gerrit2')
GERRIT_SYSTEM_GROUP = defaults.get('gerrit-system-group', 'gerrit2')
gerrit = gerritlib.gerrit.Gerrit('localhost',
GERRIT_USER,
GERRIT_PORT,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in configs[1]:
project = section['project']
options = section.get('options', dict())
description = section.get('description', None)
homepage = section.get('homepage', defaults.get('homepage', None))
upstream = section.get('upstream', None)
project_git = "%s.git" % project
project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
if 'has-github' in options or default_has_github:
create_github_project(defaults, options, project,
description, homepage)
remote_url = "ssh://localhost:%s/%s" % (GERRIT_PORT, project)
if project not in project_list:
tmpdir = tempfile.mkdtemp()
try:
repo_path = os.path.join(tmpdir, 'repo')
if upstream:
run_command("git clone %(upstream)s %(repo_path)s" %
dict(upstream=upstream,
repo_path=repo_path))
git_command(repo_path,
"fetch origin "
"+refs/heads/*:refs/copy/heads/*",
env=ssh_env)
push_string = "push %s +refs/copy/heads/*:refs/heads/*"
else:
run_command("git init %s" % repo_path)
with open(os.path.join(repo_path,
".gitreview"),
'w') as gitreview:
gitreview.write("""[gerrit]
host=%s
port=%s
project=%s
""" % (GERRIT_HOST, GERRIT_PORT, project_git))
git_command(repo_path, "add .gitreview")
cmd = ("commit -a -m'Added .gitreview' --author='%s'"
% GERRIT_GITID)
git_command(repo_path, cmd)
push_string = "push --all %s"
gerrit.createProject(project)
if not os.path.exists(project_dir):
run_command("git --bare init %s" % project_dir)
run_command("chown -R %s:%s %s"
% (GERRIT_SYSTEM_USER, GERRIT_SYSTEM_GROUP,
project_dir))
git_command(repo_path,
push_string % remote_url,
env=ssh_env)
git_command(repo_path,
"push --tags %s" % remote_url, env=ssh_env)
finally:
run_command("rm -fr %s" % tmpdir)
try:
acl_config = section.get('acl-config',
'%s.config' % os.path.join(ACL_DIR,
project))
except AttributeError:
acl_config = None
if acl_config:
if not os.path.isfile(acl_config):
write_acl_config(project,
ACL_DIR,
section.get('acl-base', None),
section.get('acl-append', []),
section.get('acl-parameters', {}))
tmpdir = tempfile.mkdtemp()
try:
repo_path = os.path.join(tmpdir, 'repo')
ret, _ = run_command_status("git init %s" % repo_path)
if ret != 0:
continue
if (fetch_config(project,
remote_url,
repo_path,
ssh_env) and
copy_acl_config(project, repo_path,
acl_config) and
create_groups_file(project, gerrit, repo_path)):
push_acl_config(project,
remote_url,
repo_path,
GERRIT_GITID,
ssh_env)
finally:
run_command("rm -fr %s" % tmpdir)
finally:
os.unlink(ssh_env['GIT_SSH'])
if __name__ == "__main__":
main()

146
jeepyb/cmd/notify_impact.py
View File

@ -1,146 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is designed to be called by a gerrit hook. It searched new
# patchsets for strings like "bug FOO" and updates corresponding Launchpad
# bugs status.
import argparse
import os
import re
import smtplib
import subprocess
from email.mime import text
from launchpadlib import launchpad
from launchpadlib import uris
BASE_DIR = '/home/gerrit2/review_site'
EMAIL_TEMPLATE = """
Hi, I'd like you to take a look at this patch for potential
%s.
%s
Log:
%s
"""
GERRIT_CACHE_DIR = os.path.expanduser(
os.environ.get('GERRIT_CACHE_DIR',
'~/.launchpadlib/cache'))
GERRIT_CREDENTIALS = os.path.expanduser(
os.environ.get('GERRIT_CREDENTIALS',
'~/.launchpadlib/creds'))
def create_bug(git_log, args, lp_project):
"""Create a bug for a change.
Create a launchpad bug in lp_project, titled with the first line of
the git commit message, with the content of the git_log prepended
with the Gerrit review URL. Tag the bug with the name of the repository
it came from. Don't create a duplicate bug. Returns link to the bug.
"""
lpconn = launchpad.Launchpad.login_with(
'Gerrit User Sync',
uris.LPNET_SERVICE_ROOT,
GERRIT_CACHE_DIR,
credentials_file=GERRIT_CREDENTIALS,
version='devel')
lines_in_log = git_log.split("\n")
bug_title = lines_in_log[4]
bug_descr = args.change_url + '\n' + git_log
project = lpconn.projects[lp_project]
# check for existing bugs by searching for the title, to avoid
# creating multiple bugs per review
potential_dupes = project.searchTasks(search_text=bug_title)
if len(potential_dupes) == 0:
buginfo = lpconn.bugs.createBug(
target=project, title=bug_title,
description=bug_descr, tags=args.project.split('/')[1])
buglink = buginfo.web_link
return buglink
def process_impact(git_log, args):
"""Process DocImpact flag.
If the 'DocImpact' flag is present, create a new documentation bug in
the openstack-manuals launchpad project based on the git_log.
For non-documentation impacts notify the mailing list of impact.
"""
if args.impact.lower() == 'docimpact':
create_bug(git_log, args, 'openstack-manuals')
return
email_content = EMAIL_TEMPLATE % (args.impact,
args.change_url, git_log)
msg = text.MIMEText(email_content)
msg['Subject'] = '[%s] %s review request change %s' % \
(args.project, args.impact, args.change)
msg['From'] = 'gerrit2@review.openstack.org'
msg['To'] = args.dest_address
s = smtplib.SMTP('localhost')
s.sendmail('gerrit2@review.openstack.org',
args.dest_address, msg.as_string())
s.quit()
def impacted(git_log, impact_string):
"""Determine if a changes log indicates there is an impact."""
return re.search(impact_string, git_log, re.IGNORECASE)
def extract_git_log(args):
"""Extract git log of all merged commits."""
cmd = ['git',
'--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
'log', '--no-merges', args.commit + '^1..' + args.commit]
return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('hook')
#common
parser.add_argument('--change', default=None)
parser.add_argument('--change-url', default=None)
parser.add_argument('--project', default=None)
parser.add_argument('--branch', default=None)
parser.add_argument('--commit', default=None)
#change-merged
parser.add_argument('--submitter', default=None)
#patchset-created
parser.add_argument('--uploader', default=None)
parser.add_argument('--patchset', default=None)
# Not passed by gerrit:
parser.add_argument('--impact', default=None)
parser.add_argument('--dest-address', default=None)
args = parser.parse_args()
# Get git log
git_log = extract_git_log(args)
# Process impacts found in git log
if impacted(git_log, args.impact):
process_impact(git_log, args)
if __name__ == "__main__":
main()

176
jeepyb/cmd/openstackwatch.py
View File

@ -1,176 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2013 Chmouel Boudjnah, eNovance
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This script is designed to generate rss feeds for subscription from updates
# to various gerrit tracked projects. It is intended to be run periodically,
# for example hourly via cron. It takes an optional argument to specify the
# path to a configuration file.
# -*- encoding: utf-8 -*-
from __future__ import print_function
__author__ = "Chmouel Boudjnah <chmouel@chmouel.com>"
import ConfigParser
import cStringIO
import datetime
import json
import os
import sys
import time
import urllib
import PyRSS2Gen
PROJECTS = ['openstack/nova', 'openstack/keystone', 'opensack/swift']
JSON_URL = 'https://review.openstack.org/query'
DEBUG = False
OUTPUT_MODE = 'multiple'
curdir = os.path.dirname(os.path.realpath(sys.argv[0]))
class ConfigurationError(Exception):
pass
def get_config(config, section, option, default=None):
if not config.has_section(section):
raise ConfigurationError("Invalid configuration, missing section: %s" %
section)
if config.has_option(section, option):
return config.get(section, option)
elif not default is None:
return default
else:
raise ConfigurationError("Invalid configuration, missing "
"section/option: %s/%s" % (section, option))
def parse_ini(inifile):
ret = {}
if not os.path.exists(inifile):
return
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.read(inifile)
if config.has_section('swift'):
ret['swift'] = dict(config.items('swift'))
ret['projects'] = get_config(config, 'general', 'projects', PROJECTS)
if type(ret['projects']) is not list:
ret['projects'] = [x.strip() for x in ret['projects'].split(',')]
ret['json_url'] = get_config(config, 'general', 'json_url', JSON_URL)
ret['debug'] = get_config(config, 'general', 'debug', DEBUG)
ret['output_mode'] = get_config(config, 'general', 'output_mode',
OUTPUT_MODE)
return ret
try:
conffile = sys.argv[1]
except IndexError:
conffile = os.path.join(curdir, '..', 'config', 'openstackwatch.ini')
CONFIG = parse_ini(conffile)
def debug(msg):
if DEBUG:
print(msg)
def get_javascript(project=None):
url = CONFIG['json_url']
if project:
url += "+project:" + project
fp = urllib.urlretrieve(url)
ret = open(fp[0]).read()
return ret
def parse_javascript(javascript):
for row in javascript.splitlines():
try:
json_row = json.loads(row)
except(ValueError):
continue
if not json_row or not 'project' in json_row or \
json_row['project'] not in CONFIG['projects']:
continue
yield json_row
def upload_rss(xml, output_object):
if 'swift' not in CONFIG:
print(xml)
return
import swiftclient
cfg = CONFIG['swift']
client = swiftclient.Connection(cfg['auth_url'],
cfg['username'],
cfg['password'],
auth_version=cfg.get('auth_version',
'2.0'))
try:
client.get_container(cfg['container'])
except(swiftclient.client.ClientException):
client.put_container(cfg['container'])
# eventual consistenties
time.sleep(1)
client.put_object(cfg['container'], output_object,
cStringIO.StringIO(xml))
def generate_rss(javascript, project=""):
title = "OpenStack %s watch RSS feed" % (project)
rss = PyRSS2Gen.RSS2(
title=title,
link="http://github.com/chmouel/openstackwatch.rss",
description="The latest reviews about Openstack, straight "
"from Gerrit.",
lastBuildDate=datetime.datetime.now()
)
for row in parse_javascript(javascript):
author = row['owner']['name']
author += " <%s>" % ('email' in row['owner'] and
row['owner']['email']
or row['owner']['username'])
rss.items.append(
PyRSS2Gen.RSSItem(
title="%s [%s]: %s" % (os.path.basename(row['project']),
row['status'],
row['subject']),
author=author,
link=row['url'],
guid=PyRSS2Gen.Guid(row['id']),
description=row['subject'],
pubDate=datetime.datetime.fromtimestamp(row['lastUpdated']),
))
return rss.to_xml()
def main():
if CONFIG['output_mode'] == "combined":
upload_rss(generate_rss(get_javascript()),
CONFIG['swift']['combined_output_object'])
elif CONFIG['output_mode'] == "multiple":
for project in CONFIG['projects']:
upload_rss(
generate_rss(get_javascript(project), project=project),
"%s.xml" % (os.path.basename(project)))
if __name__ == '__main__':
main()

286
jeepyb/cmd/trivial_rebase.py
View File

@ -1,286 +0,0 @@
# Copyright (c) 2010, Code Aurora Forum. All rights reserved.
# Copyright (c) 2012, Hewlett-Packard Development Company, L.P.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# # Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# # Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
# # Neither the name of Code Aurora Forum, Inc. nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# This script is designed to detect when a patchset uploaded to Gerrit is
# 'identical' (determined via git-patch-id) and reapply reviews onto the new
# patchset from the previous patchset.
# Get usage and help info by running: trivial-rebase --help
# Documentation is available here:
# https://www.codeaurora.org/xwiki/bin/QAEP/Gerrit
from __future__ import print_function
import json
import optparse
import subprocess
import sys
class SilentOptionParser(optparse.OptionParser):
"""Make OptionParser silently swallow unrecognized options."""
def _process_args(self, largs, rargs, values):
while rargs:
try:
optparse.OptionParser._process_args(self, largs, rargs, values)
except (optparse.AmbiguousOptionError,
optparse.BadOptionError) as e:
largs.append(e.opt_str)
class CheckCallError(OSError):
"""CheckCall() returned non-0."""
def __init__(self, command, cwd, retcode, stdout, stderr=None):
OSError.__init__(self, command, cwd, retcode, stdout, stderr)
self.command = command
self.cwd = cwd
self.retcode = retcode
self.stdout = stdout
self.stderr = stderr
def CheckCall(command, cwd=None):
"""Like subprocess.check_call() but returns stdout.
Works on python 2.4
"""
try:
process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE)
std_out, std_err = process.communicate()
except OSError as e:
raise CheckCallError(command, cwd, e.errno, None)
if process.returncode:
raise CheckCallError(command, cwd, process.returncode,
std_out, std_err)
return std_out, std_err
def Gssh(options, api_command):
"""Makes a Gerrit API call via SSH and returns the stdout results."""
ssh_cmd = ['ssh',
'-l', 'Gerrit Code Review',
'-p', options.port,
'-i', options.private_key_path,
options.server,
api_command]
try:
return CheckCall(ssh_cmd)[0]
except CheckCallError as e:
err_template = "call: %s\nreturn code: %s\nstdout: %s\nstderr: %s\n"
sys.stderr.write(err_template % (ssh_cmd,
e.retcode,
e.stdout,
e.stderr))
raise
def GsqlQuery(sql_query, options):
"""Runs a gerrit gsql query and returns the result."""
gsql_cmd = "gerrit gsql --format JSON -c %s" % sql_query
gsql_out = Gssh(options, gsql_cmd)
new_out = gsql_out.replace('}}\n', '}}\nsplit here\n')
return new_out.split('split here\n')
def FindPrevRev(options):
"""Finds the revision of the previous patch set on the change."""
sql_query = ("\"SELECT revision FROM patch_sets,changes WHERE "
"patch_sets.change_id = changes.change_id AND "
"patch_sets.patch_set_id = %s AND "
"changes.change_key = \'%s\'\"" % ((options.patchset - 1),
options.changeId))
revisions = GsqlQuery(sql_query, options)
json_dict = json.loads(revisions[0], strict=False)
return json_dict["columns"]["revision"]
def GetApprovals(options):
"""Get all the approvals on a specific patch set.
Returns a list of approval dicts
"""
sql_query = ("\"SELECT value,account_id,category_id"
" FROM patch_set_approvals"
" WHERE patch_set_id = %s"
" AND change_id = (SELECT change_id FROM"
" changes WHERE change_key = \'%s\') AND value <> 0\""
% ((options.patchset - 1), options.changeId))
gsql_out = GsqlQuery(sql_query, options)
approvals = []
for json_str in gsql_out:
dict = json.loads(json_str, strict=False)
if dict["type"] == "row":
approvals.append(dict["columns"])
return approvals
def GetPatchId(revision, consider_whitespace=False):
git_show_cmd = ['git', 'show', revision]
patch_id_cmd = ['git', 'patch-id']
patch_id_process = subprocess.Popen(patch_id_cmd, stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
git_show_process = subprocess.Popen(git_show_cmd, stdout=subprocess.PIPE)
if consider_whitespace:
# This matches on change lines in the patch (those starting with "+"
# or "-" but not followed by another of the same), then replaces any
# space or tab characters with "%" before calculating a patch-id.
replace_ws_cmd = ['sed', r'/^\(+[^+]\|-[^-]\)/y/ \t/%%/']
replace_ws_process = subprocess.Popen(replace_ws_cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
git_show_output = git_show_process.communicate()[0]
replace_ws_output = replace_ws_process.communicate(git_show_output)[0]
return patch_id_process.communicate(replace_ws_output)[0]
else:
return patch_id_process.communicate(
git_show_process.communicate()[0])[0]
def SuExec(options, as_user, cmd):
suexec_cmd = "suexec --as %s -- %s" % (as_user, cmd)
Gssh(options, suexec_cmd)
def DiffCommitMessages(commit1, commit2):
log_cmd1 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"',
commit1 + '^!']
commit1_log = CheckCall(log_cmd1)
log_cmd2 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"',
commit2 + '^!']
commit2_log = CheckCall(log_cmd2)
if commit1_log != commit2_log:
return True
return False
def main():
usage = "usage: %prog <required options> [optional options]"
parser = SilentOptionParser(usage=usage)
parser.add_option("--change", dest="changeId", help="Change identifier")
parser.add_option("--project", help="Project path in Gerrit")
parser.add_option("--commit", help="Git commit-ish for this patchset")
parser.add_option("--patchset", type="int", help="The patchset number")
parser.add_option("--role-user", dest="role_user",
help="E-mail/ID of user commenting on commit messages")
parser.add_option("--private-key-path", dest="private_key_path",
help="Full path to Gerrit SSH daemon's private host key")
parser.add_option("--server-port", dest="port", default='29418',
help="Port to connect to Gerrit's SSH daemon "
"[default: %default]")
parser.add_option("--server", dest="server", default="localhost",
help="Server name/address for Gerrit's SSH daemon "
"[default: %default]")
parser.add_option("--whitespace", action="store_true",
help="Treat whitespace as significant")
(options, args) = parser.parse_args()
if not options.changeId:
parser.print_help()
sys.exit(0)
if options.patchset == 1:
# Nothing to detect on first patchset
sys.exit(0)
prev_revision = None
prev_revision = FindPrevRev(options)
if not prev_revision:
# Couldn't find a previous revision
sys.exit(0)
prev_patch_id = GetPatchId(prev_revision)
cur_patch_id = GetPatchId(options.commit)
if cur_patch_id.split()[0] != prev_patch_id.split()[0]:
# patch-ids don't match
sys.exit(0)
# Patch ids match. This is a trivial rebase.
# In addition to patch-id we should check if whitespace content changed.
# Some languages are more sensitive to whitespace than others, and some
# changes may either introduce or be intended to fix style problems
# specifically involving whitespace as well.
if options.whitespace:
prev_patch_ws = GetPatchId(prev_revision, consider_whitespace=True)
cur_patch_ws = GetPatchId(options.commit, consider_whitespace=True)
if cur_patch_ws.split()[0] != prev_patch_ws.split()[0]:
# Insert a comment into the change letting the approvers know
# only the whitespace changed
comment_msg = ("\"New patchset patch-id matches previous patchset,"
" but whitespace content has changed.\"")
comment_cmd = ['gerrit', 'approve', '--project', options.project,
'--message', comment_msg, options.commit]
SuExec(options, options.role_user, ' '.join(comment_cmd))
sys.exit(0)
# We should also check if the commit message changed. Most approvers would
# want to re-review changes when the commit message changes.
changed = DiffCommitMessages(prev_revision, options.commit)
if changed:
# Insert a comment into the change letting the approvers know only the
# commit message changed
comment_msg = ("\"New patchset patch-id matches previous patchset,"
" but commit message has changed.\"")
comment_cmd = ['gerrit', 'approve', '--project', options.project,
'--message', comment_msg, options.commit]
SuExec(options, options.role_user, ' '.join(comment_cmd))
sys.exit(0)
# Need to get all approvals on prior patch set, then suexec them onto
# this patchset.
approvals = GetApprovals(options)
gerrit_approve_msg = ("\'Automatically re-added by Gerrit trivial rebase "
"detection script.\'")
for approval in approvals:
# Note: Sites with different 'copy_min_score' values in the
# approval_categories DB table might want different behavior here.
# Additional categories should also be added if desired.
if approval["category_id"] == "CRVW":
approve_category = '--code-review'
elif approval["category_id"] == "VRIF":
# Don't re-add verifies
#approve_category = '--verified'
continue
elif approval["category_id"] == "SUBM":
# We don't care about previous submit attempts
continue
elif approval["category_id"] == "APRV":
# Similarly squash old approvals
continue
else:
print("Unsupported category: %s" % approval)
sys.exit(0)
score = approval["value"]
gerrit_approve_cmd = ['gerrit', 'approve',
'--project', options.project,
'--message', gerrit_approve_msg,
approve_category, score, options.commit]
SuExec(options, approval["account_id"], ' '.join(gerrit_approve_cmd))
sys.exit(0)
if __name__ == "__main__":
main()

158
jeepyb/cmd/update_blueprint.py
View File

@ -1,158 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is designed to be called by a gerrit hook. It searched new
# patchsets for strings like "blueprint FOO" or "bp FOO" and updates
# corresponding Launchpad blueprints with links back to the change.
import argparse
import ConfigParser
import os
import re
import StringIO
import subprocess
from launchpadlib import launchpad
from launchpadlib import uris
import MySQLdb
BASE_DIR = '/home/gerrit2/review_site'
GERRIT_CACHE_DIR = os.path.expanduser(
os.environ.get('GERRIT_CACHE_DIR',
'~/.launchpadlib/cache'))
GERRIT_CREDENTIALS = os.path.expanduser(
os.environ.get('GERRIT_CREDENTIALS',
'~/.launchpadlib/creds'))
GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG',
'/home/gerrit2/review_site/etc/gerrit.config')
GERRIT_SECURE_CONFIG_DEFAULT = '/home/gerrit2/review_site/etc/secure.config'
GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG',
GERRIT_SECURE_CONFIG_DEFAULT)
SPEC_RE = re.compile(r'\b(blueprint|bp)\b[ \t]*[#:]?[ \t]*(\S+)', re.I)
BODY_RE = re.compile(r'^\s+.*$')
def get_broken_config(filename):
"""gerrit config ini files are broken and have leading tabs."""
text = ""
with open(filename, "r") as conf:
for line in conf.readlines():
text = "%s%s" % (text, line.lstrip())
fp = StringIO.StringIO(text)
c = ConfigParser.ConfigParser()
c.readfp(fp)
return c
GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG)
SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG)
DB_USER = GERRIT_CONFIG.get("database", "username")
DB_PASS = SECURE_CONFIG.get("database", "password")
DB_DB = GERRIT_CONFIG.get("database", "database")
def short_project(full_project_name):
"""Return the project part of the git repository name."""
return full_project_name.split('/')[-1]
def git2lp(full_project_name):
"""Convert Git repo name to Launchpad project."""
project_map = {
'stackforge/puppet-openstack_dev_env': 'puppet-openstack',
'stackforge/puppet-quantum': 'puppet-neutron',
}
return project_map.get(full_project_name, short_project(full_project_name))
def update_spec(launchpad, project, name, subject, link, topic=None):
project = git2lp(project)
spec = launchpad.projects[project].getSpecification(name=name)
if not spec:
return
if spec.whiteboard:
wb = spec.whiteboard.strip()
else:
wb = ''
changed = False
if topic:
topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/', 8)],
topic)
if topiclink not in wb:
wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink)
changed = True
if link not in wb:
wb += ("\n\n\nAddressed by: {link}\n"
" {subject}\n").format(subject=subject,
link=link)
changed = True
if changed:
spec.whiteboard = wb
spec.lp_save()
def find_specs(launchpad, dbconn, args):
git_dir_arg = '--git-dir={base_dir}/git/{project}.git'.format(
base_dir=BASE_DIR,
project=args.project)
git_log = subprocess.Popen(['git', git_dir_arg, 'log', '--no-merges',
args.commit + '^1..' + args.commit],
stdout=subprocess.PIPE).communicate()[0]
cur = dbconn.cursor()
cur.execute("select subject, topic from changes where change_key=%s",
args.change)
subject, topic = cur.fetchone()
specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)])
if topic:
topicspec = topic.split('/')[-1]
specs |= set([topicspec])
for spec in specs:
update_spec(launchpad, args.project, spec, subject,
args.change_url, topic)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('hook')
#common
parser.add_argument('--change', default=None)
parser.add_argument('--change-url', default=None)
parser.add_argument('--project', default=None)
parser.add_argument('--branch', default=None)
parser.add_argument('--commit', default=None)
#change-merged
parser.add_argument('--submitter', default=None)
# patchset-created
parser.add_argument('--uploader', default=None)
parser.add_argument('--patchset', default=None)
args = parser.parse_args()
lpconn = launchpad.Launchpad.login_with(
'Gerrit User Sync', uris.LPNET_SERVICE_ROOT, GERRIT_CACHE_DIR,
credentials_file=GERRIT_CREDENTIALS, version='devel')
conn = MySQLdb.connect(user=DB_USER, passwd=DB_PASS, db=DB_DB)
find_specs(lpconn, conn, args)
if __name__ == "__main__":
main()

405
jeepyb/cmd/update_bug.py
View File

@ -1,405 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This is designed to be called by a gerrit hook. It searched new
# patchsets for strings like "bug FOO" and updates corresponding Launchpad
# bugs status.
import argparse
import os
import re
import subprocess
from launchpadlib import launchpad
from launchpadlib import uris
import jeepyb.gerritdb
BASE_DIR = '/home/gerrit2/review_site'
GERRIT_CACHE_DIR = os.path.expanduser(
os.environ.get('GERRIT_CACHE_DIR',
'~/.launchpadlib/cache'))
GERRIT_CREDENTIALS = os.path.expanduser(
os.environ.get('GERRIT_CREDENTIALS',
'~/.launchpadlib/creds'))
def add_change_proposed_message(bugtask, change_url, project, branch):
subject = 'Fix proposed to %s (%s)' % (short_project(project), branch)
body = 'Fix proposed to branch: %s\nReview: %s' % (branch, change_url)
bugtask.bug.newMessage(subject=subject, content=body)
def add_change_merged_message(bugtask, change_url, project, commit,
submitter, branch, git_log):
subject = 'Fix merged to %s (%s)' % (short_project(project), branch)
git_url = 'http://github.com/%s/commit/%s' % (project, commit)
body = '''Reviewed: %s
Committed: %s
Submitter: %s
Branch: %s\n''' % (change_url, git_url, submitter, branch)
body = body + '\n' + git_log
bugtask.bug.newMessage(subject=subject, content=body)
def set_in_progress(bugtask, launchpad, uploader, change_url):
"""Set bug In progress with assignee being the uploader"""
# Retrieve uploader from Launchpad by correlating Gerrit E-mail
# address to OpenID, and only set if there is a clear match.
try:
searchkey = uploader[uploader.rindex("(") + 1:-1]
except ValueError:
searchkey = uploader
# The counterintuitive query is due to odd database schema choices
# in Gerrit. For example, an account with a secondary E-mail
# address added looks like...
# select email_address,external_id from account_external_ids
# where account_id=1234;
# +-----------------+-----------------------------------------+
# | email_address | external_id |
# +-----------------+-----------------------------------------+
# | plugh@xyzzy.com | https://login.launchpad.net/+id/fR0bnU1 |
# | bar@foo.org | mailto:bar@foo.org |
# | NULL | username:quux |
# +-----------------+-----------------------------------------+
# ...thus we need a join on a secondary query to search against
# all the user's configured E-mail addresses.
#
query = """SELECT t.external_id FROM account_external_ids t
INNER JOIN (
SELECT t.account_id FROM account_external_ids t
WHERE t.email_address = %s )
original ON t.account_id = original.account_id
AND t.external_id LIKE 'https://login.launchpad.net%%'"""
cursor = jeepyb.gerritdb.connect().cursor()
cursor.execute(query, searchkey)
data = cursor.fetchone()
if data:
assignee = launchpad.people.getByOpenIDIdentifier(identifier=data[0])
if assignee:
bugtask.assignee = assignee
bugtask.status = "In Progress"
bugtask.lp_save()
def set_fix_committed(bugtask):
"""Set bug fix committed."""
bugtask.status = "Fix Committed"
bugtask.lp_save()
def set_fix_released(bugtask):
"""Set bug fix released."""
bugtask.status = "Fix Released"
bugtask.lp_save()
def release_fixcommitted(bugtask):
"""Set bug FixReleased if it was FixCommitted."""
if bugtask.status == u'Fix Committed':
set_fix_released(bugtask)
def tag_in_branchname(bugtask, branch):
"""Tag bug with in-branch-name tag (if name is appropriate)."""
lp_bug = bugtask.bug
branch_name = branch.replace('/', '-')
if branch_name.replace('-', '').isalnum():
lp_bug.tags = lp_bug.tags + ["in-%s" % branch_name]
lp_bug.tags.append("in-%s" % branch_name)
lp_bug.lp_save()
def short_project(full_project_name):
"""Return the project part of the git repository name."""
return full_project_name.split('/')[-1]
def git2lp(full_project_name):
"""Convert Git repo name to Launchpad project."""
project_map = {
'openstack/api-site': 'openstack-api-site',
'openstack/quantum': 'neutron',
'openstack/python-quantumclient': 'python-neutronclient',
'openstack/oslo-incubator': 'oslo',
'openstack/tripleo-incubator': 'tripleo',
'openstack-infra/askbot-theme': 'openstack-ci',
'openstack-infra/config': 'openstack-ci',
'openstack-infra/devstack-gate': 'openstack-ci',
'openstack-infra/gear': 'openstack-ci',
'openstack-infra/gerrit': 'openstack-ci',
'openstack-infra/gerritbot': 'openstack-ci',
'openstack-infra/gerritlib': 'openstack-ci',
'openstack-infra/gitdm': 'openstack-ci',
'openstack-infra/jeepyb': 'openstack-ci',
'openstack-infra/jenkins-job-builder': 'openstack-ci',
'openstack-infra/lodgeit': 'openstack-ci',
'openstack-infra/meetbot': 'openstack-ci',
'openstack-infra/nose-html-output': 'openstack-ci',
'openstack-infra/publications': 'openstack-ci',
'openstack-infra/puppet-apparmor': 'openstack-ci',
'openstack-infra/puppet-dashboard': 'openstack-ci',
'openstack-infra/puppet-vcsrepo': 'openstack-ci',
'openstack-infra/reviewday': 'openstack-ci',
'openstack-infra/statusbot': 'openstack-ci',
'openstack-infra/zmq-event-publisher': 'openstack-ci',
'stackforge/cookbook-openstack-block-storage': 'openstack-chef',
'stackforge/cookbook-openstack-common': 'openstack-chef',
'stackforge/cookbook-openstack-compute': 'openstack-chef',
'stackforge/cookbook-openstack-dashboard': 'openstack-chef',
'stackforge/cookbook-openstack-identity': 'openstack-chef',
'stackforge/cookbook-openstack-image': 'openstack-chef',
'stackforge/cookbook-openstack-metering': 'openstack-chef',
'stackforge/cookbook-openstack-network': 'openstack-chef',
'stackforge/cookbook-openstack-object-storage': 'openstack-chef',
'stackforge/cookbook-openstack-ops-database': 'openstack-chef',
'stackforge/cookbook-openstack-ops-messaging': 'openstack-chef',
'stackforge/cookbook-openstack-orchestration': 'openstack-chef',
'stackforge/openstack-chef-repo': 'openstack-chef',
'stackforge/puppet-openstack_dev_env': 'puppet-openstack',
'stackforge/puppet-quantum': 'puppet-neutron',
'stackforge/tripleo-heat-templates': 'tripleo',
'stackforge/tripleo-image-elements': 'tripleo',
}
return project_map.get(full_project_name, short_project(full_project_name))
def is_direct_release(full_project_name):
"""Test against a list of projects who directly release changes."""
return full_project_name in [
'openstack/openstack-manuals',
'openstack/api-site',
'openstack/tripleo-incubator',
'openstack-dev/devstack',
'openstack-infra/askbot-theme',
'openstack-infra/config',
'openstack-infra/devstack-gate',
'openstack-infra/gerrit',
'openstack-infra/gerritbot',
'openstack-infra/gerritlib',
'openstack-infra/gitdm',
'openstack-infra/lodgeit',
'openstack-infra/meetbot',
'openstack-infra/nose-html-output',
'openstack-infra/publications',
'openstack-infra/reviewday',
'openstack-infra/statusbot',
'stackforge/cookbook-openstack-block-storage',
'stackforge/cookbook-openstack-common',
'stackforge/cookbook-openstack-compute',
'stackforge/cookbook-openstack-dashboard',
'stackforge/cookbook-openstack-identity',
'stackforge/cookbook-openstack-image',
'stackforge/cookbook-openstack-metering',
'stackforge/cookbook-openstack-network',
'stackforge/cookbook-openstack-object-storage',
'stackforge/cookbook-openstack-ops-database',
'stackforge/cookbook-openstack-ops-messaging',
'stackforge/cookbook-openstack-orchestration',
'stackforge/openstack-chef-repo',
'stackforge/tripleo-heat-templates',
'stackforge/tripleo-image-elements',
]
class Task:
def __init__(self, lp_task, prefix):
'''Prefixes associated with bug references will allow for certain
changes to be made to the bug's launchpad (lp) page. The following
tokens represent the automation currently taking place.
::
add_comment -> Adds a comment to the bug's lp page.
set_in_progress -> Sets the bug's lp status to 'In Progress'.
set_fix_released -> Sets the bug's lp status to 'Fix Released'.
set_fix_committed -> Sets the bug's lp status to 'Fix Committed'.
::
changes_needed, when populated, simply indicates the actions that are
available to be taken based on the value of 'prefix'.
'''
self.lp_task = lp_task
self.changes_needed = []
# If no prefix was matched, default to 'closes'.
prefix = prefix.split('-')[0].lower() if prefix else 'closes'
if prefix in ('closes', 'fixes', 'resolves'):
self.changes_needed.extend(('add_comment',
'set_in_progress',
'set_fix_committed',
'set_fix_released'))
elif prefix in ('partial',):
self.changes_needed.extend(('add_comment', 'set_in_progress'))
elif prefix in ('related', 'impacts', 'affects'):
self.changes_needed.extend(('add_comment',))
else:
# prefix is not recognized.
self.changes_needed.extend(('add_comment',))
def needs_change(self, change):
'''Return a boolean indicating if given 'change' needs to be made.'''
if change in self.changes_needed:
return True
else:
return False
def process_bugtask(launchpad, task, git_log, args):
"""Apply changes to lp bug tasks, based on hook / branch."""
bugtask = task.lp_task
if args.hook == "change-merged":
if args.branch == 'master':
if (is_direct_release(args.project) and
task.needs_change('set_fix_released')):
set_fix_released(bugtask)
else:
if (bugtask.status != u'Fix Released' and
task.needs_change('set_fix_committed')):
set_fix_committed(bugtask)
elif args.branch == 'milestone-proposed':
release_fixcommitted(bugtask)
elif args.branch.startswith('stable/'):
series = args.branch[7:]
# Look for a related task matching the series.
for reltask in bugtask.related_tasks:
if (reltask.bug_target_name.endswith("/" + series) and
reltask.status != u'Fix Released' and
task.needs_change('set_fix_committed')):
set_fix_committed(reltask)
break
else:
# Use tag_in_branchname if there isn't any.
tag_in_branchname(bugtask, args.branch)
add_change_merged_message(bugtask, args.change_url, args.project,
args.commit, args.submitter, args.branch,
git_log)
if args.hook == "patchset-created":
if args.branch == 'master':
if (bugtask.status not in [u'Fix Committed', u'Fix Released'] and
task.needs_change('set_in_progress')):
set_in_progress(bugtask, launchpad,
args.uploader, args.change_url)
elif args.branch.startswith('stable/'):
series = args.branch[7:]
for reltask in bugtask.related_tasks:
if (reltask.bug_target_name.endswith("/" + series) and
task.needs_change('set_in_progress') and
reltask.status not in [u'Fix Committed',
u'Fix Released']):
set_in_progress(reltask, launchpad,
args.uploader, args.change_url)
break
if args.patchset == '1':
add_change_proposed_message(bugtask, args.change_url,
args.project, args.branch)
def find_bugs(launchpad, git_log, args):
'''Find bugs referenced in the git log and return related tasks.
Our regular expression is composed of three major parts:
part1: Matches only at start-of-line (required). Optionally matches any
word or hyphen separated words.
part2: Matches the words 'bug' or 'lp' on a word boundry (required).
part3: Matches a whole number (required).
The following patterns will be matched properly:
bug # 555555
Closes-Bug: 555555
Fixes: bug # 555555
Resolves: bug 555555
Partial-Bug: lp bug # 555555
:returns: an iterable containing Task objects.
'''
part1 = r'^[\t ]*(?P<prefix>[-\w]+)?[\s:]*'
part2 = r'(?:\b(?:bug|lp)\b[\s#:]*)+'
part3 = r'(?P<bug_number>\d+)\s*?$'
regexp = part1 + part2 + part3
matches = re.finditer(regexp, git_log, flags=re.I | re.M)
# Extract unique bug tasks and associated prefixes.
bugtasks = {}
for match in matches:
prefix = match.group('prefix')
bug_num = match.group('bug_number')
if bug_num not in bugtasks:
try:
lp_bug = launchpad.bugs[bug_num]
for lp_task in lp_bug.bug_tasks:
if lp_task.bug_target_name == git2lp(args.project):
bugtasks[bug_num] = Task(lp_task, prefix)
break
except KeyError:
# Unknown bug.
pass
return bugtasks.values()
def extract_git_log(args):
"""Extract git log of all merged commits."""
cmd = ['git',
'--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
'log', '--no-merges', args.commit + '^1..' + args.commit]
return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('hook')
# common
parser.add_argument('--change', default=None)
parser.add_argument('--change-url', default=None)
parser.add_argument('--project', default=None)
parser.add_argument('--branch', default=None)
parser.add_argument('--commit', default=None)
# change-merged
parser.add_argument('--submitter', default=None)
# patchset-created
parser.add_argument('--uploader', default=None)
parser.add_argument('--patchset', default=None)
args = parser.parse_args()
# Connect to Launchpad.
lpconn = launchpad.Launchpad.login_with(
'Gerrit User Sync', uris.LPNET_SERVICE_ROOT, GERRIT_CACHE_DIR,
credentials_file=GERRIT_CREDENTIALS, version='devel')
# Get git log.
git_log = extract_git_log(args)
# Process tasks found in git log.
for task in find_bugs(lpconn, git_log, args):
process_bugtask(lpconn, task, git_log, args)
if __name__ == "__main__":
main()

39
jeepyb/config/openstackwatch.ini-sample
View File

@ -1,39 +0,0 @@
# -*- Mode: conf -*-
[general]
# only show certain projects (don't forget the openstack/ as start)
projects = openstack/swift, openstack/cinder
# The Json URL where is the gerrit system.
json_url = https://review.openstack.org/query?q=status:open
# Allow different mode to output to swift, by default 'combined' will
# combined all rss in one and 'multiple' will upload all the projects
# in each rss file.
output_mode = multiple
# username to your swift cluster
# [swift]
# username/tenant for swift with 2.0 or just username with 1.0 (i.e:
# RAX).
# username =
# passowrd or api key
# password =
# container to upload (probably want to be public)
# container =
# auth_url of the cluster, for Rackspace this is :
# https://auth.api.rackspacecloud.com/v1.0
# or Rackspace UK :
# https://lon.auth.api.rackspacecloud.com/v1.0
# auth_url = https://lon.auth.api.rackspacecloud.com/v1.0
# auth version (1.0 for Rackspace clouds, 2.0 for keystone backend clusters)
# auth_version = 1.0
# the object name where to store the combined rss
# combined_output_object = openstackwatch.xml
# vim: ft=dosini

55
jeepyb/gerritdb.py
View File

@ -1,55 +0,0 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ConfigParser
import MySQLdb
import os
import StringIO
GERRIT_CONFIG = os.environ.get(
'GERRIT_CONFIG',
'/home/gerrit2/review_site/etc/gerrit.config')
GERRIT_SECURE_CONFIG = os.environ.get(
'GERRIT_SECURE_CONFIG',
'/home/gerrit2/review_site/etc/secure.config')
db_connection = None
def get_broken_config(filename):
"""gerrit config ini files are broken and have leading tabs."""
text = ""
for line in open(filename, "r"):
text += line.lstrip()
fp = StringIO.StringIO(text)
c = ConfigParser.ConfigParser()
c.readfp(fp)
return c
def connect():
global db_connection
if not db_connection:
gerrit_config = get_broken_config(GERRIT_CONFIG)
secure_config = get_broken_config(GERRIT_SECURE_CONFIG)
DB_USER = gerrit_config.get("database", "username")
DB_PASS = secure_config.get("database", "password")
DB_DB = gerrit_config.get("database", "database")
db_connection = MySQLdb.connect(user=DB_USER, passwd=DB_PASS, db=DB_DB)
return db_connection

0
jeepyb/__init__.py → pypi_mirror/__init__.py
View File

0
jeepyb/cmd/__init__.py → pypi_mirror/cmd/__init__.py
View File

0
jeepyb/cmd/run_mirror.py → pypi_mirror/cmd/run_mirror.py
View File

8
requirements.txt
View File

@ -1,9 +1,3 @@
gerritlib
MySQL-python
paramiko
PyGithub
pyyaml
PyYAML>=3.1.0
pkginfo
PyRSS2Gen
python-swiftclient
pip>=1.4

21
setup.cfg
View File

@ -1,6 +1,6 @@
[metadata]
name = jeepyb
summary = Tools for managing gerrit projects and external sources.
name = pypi-mirror
summary = Utility to build a partial PyPI mirror
description-file =
README.rst
author = OpenStack Infrastructure Team
@ -16,21 +16,6 @@ classifier =
Programming Language :: Python :: 2.7
Programming Language :: Python :: 2.6
[global]
setup-hooks =
pbr.hooks.setup_hook
[entry_points]
console_scripts =
close-pull-requests = jeepyb.cmd.close_pull_requests:main
create-cgitrepos = jeepyb.cmd.create_cgitrepos:main
expire-old-reviews = jeepyb.cmd.expire_old_reviews:main
fetch-remotes = jeepyb.cmd.fetch_remotes:main
manage-projects = jeepyb.cmd.manage_projects:main
notify-impact = jeepyb.cmd.notify_impact:main
openstackwatch = jeepyb.cmd.openstackwatch:main
process-cache = jeepyb.cmd.process_cache:main
run-mirror = jeepyb.cmd.run_mirror:main
trivial-rebase = jeepyb.cmd.trivial_rebase:main
update-blueprint = jeepyb.cmd.update_blueprint:main
update-bug = jeepyb.cmd.update_bug:main
run-mirror = pypi_mirror.cmd.run_mirror:main

5
setup.py
View File

@ -14,8 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
setuptools.setup(
setup_requires=['d2to1', 'pbr'],
d2to1=True)
setup_requires=['pbr'],
pbr=True)

2
test-requirements.txt
View File

@ -1 +1 @@
hacking>=0.5.6,<0.7
hacking>=0.5.6,<0.8