Consume jeepyb.
Instead of keeping many of these files directly in the tree, use them from the out-of-tree jeepyb project, which makes them easier to consume for other people who are not us. Change-Id: Id704f2e17dd80709ef63cbbf2c5475a08a835f91 Reviewed-on: https://review.openstack.org/16777 Reviewed-by: Clark Boylan <clark.boylan@gmail.com> Reviewed-by: James E. Blair <corvus@inaugust.com> Approved: James E. Blair <corvus@inaugust.com> Tested-by: Jenkins
This commit is contained in:
Monty Taylor
Jenkins
parent
c4de868747
commit
52db16762b
@ -452,7 +452,7 @@ to use this build step.
|
||||
Auto Review Expiry
|
||||
==================
|
||||
|
||||
Puppet automatically installs a daily cron job called ``expire_old_reviews.py``
|
||||
Puppet automatically installs a daily cron job called ``expire-old-reviews``
|
||||
onto the gerrit servers. This script follows two rules:
|
||||
|
||||
#. If the review hasn't been touched in 2 weeks, mark as abandoned.
|
||||
|
||||
@ -1,78 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2012 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This script is designed to expire old code reviews that have not been touched
|
||||
# using the following rules:
|
||||
# 1. if open and no activity in 2 weeks, expire
|
||||
# 2. if negative comment and no activity in 1 week, expire
|
||||
|
||||
import paramiko
|
||||
import json
|
||||
import logging
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('user', help='The gerrit admin user')
|
||||
parser.add_argument('ssh_key', help='The gerrit admin SSH key file')
|
||||
options = parser.parse_args()
|
||||
|
||||
GERRIT_USER = options.user
|
||||
GERRIT_SSH_KEY = options.ssh_key
|
||||
|
||||
logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', filename='/var/log/gerrit/expire_reviews.log')
|
||||
logger= logging.getLogger('expire_reviews')
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
logger.info('Starting expire reviews')
|
||||
logger.info('Connecting to Gerrit')
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect('localhost', username=GERRIT_USER, key_filename=GERRIT_SSH_KEY, port=29418)
|
||||
|
||||
def expire_patch_set(patch_id, patch_subject, has_negative):
|
||||
if has_negative:
|
||||
message= 'code review expired after 1 week of no activity after a negative review, it can be restored using the \`Restore Change\` button under the Patch Set on the web interface'
|
||||
else:
|
||||
message= 'code review expired after 2 weeks of no activity, it can be restored using the \`Restore Change\` button under the Patch Set on the web interface'
|
||||
command='gerrit review --abandon --message="{0}" {1}'.format(message, patch_id)
|
||||
logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message)
|
||||
stdin, stdout, stderr = ssh.exec_command(command)
|
||||
if stdout.channel.recv_exit_status() != 0:
|
||||
logger.error(stderr.read())
|
||||
|
||||
# Query all open with no activity for 2 weeks
|
||||
logger.info('Searching no activity for 2 weeks')
|
||||
stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --format JSON status:open age:2w')
|
||||
|
||||
for line in stdout:
|
||||
row= json.loads(line)
|
||||
if not row.has_key('rowCount'):
|
||||
expire_patch_set(row['currentPatchSet']['revision'], row['subject'], False)
|
||||
|
||||
# Query all reviewed with no activity for 1 week
|
||||
logger.info('Searching no activity on negative review for 1 week')
|
||||
stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --all-approvals --format JSON status:reviewed age:1w')
|
||||
|
||||
for line in stdout:
|
||||
row= json.loads(line)
|
||||
if not row.has_key('rowCount'):
|
||||
# Search for negative approvals
|
||||
for approval in row['currentPatchSet']['approvals']:
|
||||
if approval['value'] == '-1':
|
||||
expire_patch_set(row['currentPatchSet']['revision'], row['subject'], True)
|
||||
break
|
||||
|
||||
logger.info('End expire review')
|
||||
@ -1,76 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
# Copyright (C) 2011 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Fetch remotes reads a project config file called projects.yaml
|
||||
# It should look like:
|
||||
|
||||
# - homepage: http://openstack.org
|
||||
# team-id: 153703
|
||||
# has-wiki: False
|
||||
# has-issues: False
|
||||
# has-downloads: False
|
||||
# ---
|
||||
# - project: PROJECT_NAME
|
||||
# options:
|
||||
# - remote: https://gerrit.googlesource.com/gerrit
|
||||
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import shlex
|
||||
import yaml
|
||||
|
||||
def run_command(cmd, status=False, env={}):
|
||||
cmd_list = shlex.split(str(cmd))
|
||||
newenv = os.environ
|
||||
newenv.update(env)
|
||||
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, env=newenv)
|
||||
(out, nothing) = p.communicate()
|
||||
if status:
|
||||
return (p.returncode, out.strip())
|
||||
return out.strip()
|
||||
|
||||
|
||||
def run_command_status(cmd, env={}):
|
||||
return run_command(cmd, True, env)
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.ERROR)
|
||||
|
||||
REPO_ROOT = os.environ.get('REPO_ROOT',
|
||||
'/home/gerrit2/review_site/git')
|
||||
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
|
||||
'/home/gerrit2/projects.yaml')
|
||||
|
||||
(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))]
|
||||
|
||||
for section in config:
|
||||
project = section['project']
|
||||
|
||||
if 'remote' not in section:
|
||||
continue
|
||||
|
||||
project_git = "%s.git" % project
|
||||
os.chdir(os.path.join(REPO_ROOT, project_git))
|
||||
|
||||
# Make sure that the specified remote exists
|
||||
remote_url = section['remote']
|
||||
# We could check if it exists first, but we're ignoring output anyway
|
||||
# So just try to make it, and it'll either make a new one or do nothing
|
||||
run_command("git remote add -f upstream %s" % remote_url)
|
||||
# Fetch new revs from it
|
||||
run_command("git remote update upstream")
|
||||
@ -1,29 +0,0 @@
|
||||
import argparse
|
||||
import paramiko
|
||||
import json
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", dest="host", default="review.openstack.org",
|
||||
help="gerrit host to connect to")
|
||||
parser.add_argument("--port", dest="port", action='store', type=int,
|
||||
default=29418, help="gerrit port to connect to")
|
||||
parser.add_argument("groups", nargs=1)
|
||||
|
||||
options = parser.parse_args()
|
||||
|
||||
|
||||
client = paramiko.SSHClient()
|
||||
client.load_system_host_keys()
|
||||
client.set_missing_host_key_policy(paramiko.WarningPolicy())
|
||||
client.connect(options.host, port=options.port)
|
||||
|
||||
group = options.groups[0]
|
||||
query = "select group_uuid from account_groups where name = '%s'" % group
|
||||
command = 'gerrit gsql --format JSON -c "%s"' % query
|
||||
stdin, stdout, stderr = client.exec_command(command)
|
||||
|
||||
for line in stdout:
|
||||
row = json.loads(line)
|
||||
if row['type'] == 'row':
|
||||
print row['columns']['group_uuid']
|
||||
ret = stdout.channel.recv_exit_status()
|
||||
@ -1,327 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
# Copyright (C) 2011 OpenStack, LLC.
|
||||
# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# manage_projects.py reads a project config file called projects.yaml
|
||||
# It should look like:
|
||||
|
||||
# - homepage: http://openstack.org
|
||||
# gerrit-host: review.openstack.org
|
||||
# local-git-dir: /var/lib/git
|
||||
# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
|
||||
# has-wiki: False
|
||||
# has-issues: False
|
||||
# has-downloads: False
|
||||
# ---
|
||||
# - project: PROJECT_NAME
|
||||
# options:
|
||||
# - has-wiki
|
||||
# - has-issues
|
||||
# - has-downloads
|
||||
# - has-pull-requests
|
||||
# homepage: Some homepage that isn't http://openstack.org
|
||||
# description: This is a great project
|
||||
# remote: https://gerrit.googlesource.com/gerrit
|
||||
# upstream: git://github.com/bushy/beards.git
|
||||
# acl_config: /path/to/gerrit/project.config
|
||||
|
||||
|
||||
import ConfigParser
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import tempfile
|
||||
import yaml
|
||||
|
||||
import github
|
||||
import gerritlib.gerrit
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.ERROR)
|
||||
log = logging.getLogger("manage_projects")
|
||||
|
||||
|
||||
def run_command(cmd, status=False, env={}):
|
||||
cmd_list = shlex.split(str(cmd))
|
||||
newenv = os.environ
|
||||
newenv.update(env)
|
||||
log.debug("Executing command: %s" % " ".join(cmd_list))
|
||||
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, env=newenv)
|
||||
(out, nothing) = p.communicate()
|
||||
log.debug("Return code: %s" % p.returncode)
|
||||
log.debug("Command said: %s" % out.strip())
|
||||
if status:
|
||||
return (p.returncode, out.strip())
|
||||
return out.strip()
|
||||
|
||||
|
||||
def run_command_status(cmd, env={}):
|
||||
return run_command(cmd, True, env)
|
||||
|
||||
|
||||
def git_command(repo_dir, sub_cmd, env={}):
|
||||
git_dir = os.path.join(repo_dir, '.git')
|
||||
cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
|
||||
status, _ = run_command(cmd, True, env)
|
||||
return status
|
||||
|
||||
|
||||
def git_command_output(repo_dir, sub_cmd, env={}):
|
||||
git_dir = os.path.join(repo_dir, '.git')
|
||||
cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
|
||||
status, out = run_command(cmd, True, env)
|
||||
return (status, out)
|
||||
|
||||
|
||||
def fetch_config(project, remote_url, repo_path, env={}):
|
||||
status = git_command(repo_path, "fetch %s +refs/meta/config:"
|
||||
"refs/remotes/gerrit-meta/config" % remote_url, env)
|
||||
if status != 0:
|
||||
print "Failed to fetch refs/meta/config for project: %s" % project
|
||||
return False
|
||||
# Because the following fails if executed more than once you should only
|
||||
# run fetch_config once in each repo.
|
||||
status = git_command(repo_path, "checkout -b config "
|
||||
"remotes/gerrit-meta/config")
|
||||
if status != 0:
|
||||
print "Failed to checkout config for project: %s" % project
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def copy_acl_config(project, repo_path, acl_config):
|
||||
if not os.path.exists(acl_config):
|
||||
return False
|
||||
|
||||
acl_dest = os.path.join(repo_path, "project.config")
|
||||
status, _ = run_command("cp %s %s" %
|
||||
(acl_config, acl_dest), status=True)
|
||||
if status == 0:
|
||||
status = git_command(repo_path, "diff --quiet HEAD")
|
||||
if status != 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def push_acl_config(project, remote_url, repo_path, env={}):
|
||||
cmd = "commit -a -m'Update project config.' --author='Openstack Project " \
|
||||
"Creator <openstack-infra@lists.openstack.org>'"
|
||||
status = git_command(repo_path, cmd)
|
||||
if status != 0:
|
||||
print "Failed to commit config for project: %s" % project
|
||||
return False
|
||||
status, out = git_command_output(repo_path,
|
||||
"push %s HEAD:refs/meta/config" %
|
||||
remote_url, env)
|
||||
if status != 0:
|
||||
print "Failed to push config for project: %s" % project
|
||||
print out
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _get_group_uuid(gerrit, group):
|
||||
query = "select group_uuid from account_groups where name = '%s'" % group
|
||||
data = gerrit.dbQuery(query)
|
||||
if data:
|
||||
for row in data:
|
||||
if row["type"] == "row":
|
||||
return row["columns"]["group_uuid"]
|
||||
return None
|
||||
|
||||
|
||||
def get_group_uuid(gerrit, group):
|
||||
uuid = _get_group_uuid(gerrit, group)
|
||||
if uuid:
|
||||
return uuid
|
||||
gerrit.createGroup(group)
|
||||
uuid = _get_group_uuid(gerrit, group)
|
||||
if uuid:
|
||||
return uuid
|
||||
return None
|
||||
|
||||
|
||||
def create_groups_file(project, gerrit, repo_path):
|
||||
acl_config = os.path.join(repo_path, "project.config")
|
||||
group_file = os.path.join(repo_path, "groups")
|
||||
uuids = {}
|
||||
for line in open(acl_config, 'r'):
|
||||
r = re.match(r'^\s+.*group\s+(.*)$', line)
|
||||
if r:
|
||||
group = r.group(1)
|
||||
if group in uuids.keys():
|
||||
continue
|
||||
uuid = get_group_uuid(gerrit, group)
|
||||
if uuid:
|
||||
uuids[group] = uuid
|
||||
else:
|
||||
return False
|
||||
if uuids:
|
||||
with open(group_file, 'w') as fp:
|
||||
for group, uuid in uuids.items():
|
||||
fp.write("%s\t%s\n" % (uuid, group))
|
||||
status = git_command(repo_path, "add groups")
|
||||
if status != 0:
|
||||
print "Failed to add groups file for project: %s" % project
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def make_ssh_wrapper(gerrit_user, gerrit_key):
|
||||
(fd, name) = tempfile.mkstemp(text=True)
|
||||
os.write(fd, '#!/bin/bash\n')
|
||||
os.write(fd,
|
||||
'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
|
||||
(gerrit_key, gerrit_user))
|
||||
os.close(fd)
|
||||
os.chmod(name, 755)
|
||||
return dict(GIT_SSH=name)
|
||||
|
||||
|
||||
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
|
||||
'/home/gerrit2/projects.yaml')
|
||||
configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
|
||||
defaults = configs[0][0]
|
||||
default_has_issues = defaults.get('has-issues', False)
|
||||
default_has_downloads = defaults.get('has-downloads', False)
|
||||
default_has_wiki = defaults.get('has-wiki', False)
|
||||
|
||||
LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
|
||||
GERRIT_HOST = defaults.get('gerrit-host')
|
||||
GERRIT_USER = defaults.get('gerrit-user')
|
||||
GERRIT_KEY = defaults.get('gerrit-key')
|
||||
GITHUB_SECURE_CONFIG = defaults.get('github-config',
|
||||
'/etc/github/github-projects.secure.config')
|
||||
|
||||
secure_config = ConfigParser.ConfigParser()
|
||||
secure_config.read(GITHUB_SECURE_CONFIG)
|
||||
|
||||
# Project creation doesn't work via oauth
|
||||
ghub = github.Github(secure_config.get("github", "username"),
|
||||
secure_config.get("github", "password"))
|
||||
orgs = ghub.get_user().get_orgs()
|
||||
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
|
||||
|
||||
gerrit = gerritlib.gerrit.Gerrit('localhost',
|
||||
GERRIT_USER,
|
||||
29418,
|
||||
GERRIT_KEY)
|
||||
project_list = gerrit.listProjects()
|
||||
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
|
||||
try:
|
||||
|
||||
for section in configs[1]:
|
||||
project = section['project']
|
||||
options = section.get('options', dict())
|
||||
description = section.get('description', None)
|
||||
homepage = section.get('homepage', defaults.get('homepage', None))
|
||||
upstream = section.get('upstream', None)
|
||||
|
||||
project_git = "%s.git" % project
|
||||
project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
|
||||
|
||||
# Find the project's repo
|
||||
project_split = project.split('/', 1)
|
||||
if len(project_split) > 1:
|
||||
repo_name = project_split[1]
|
||||
else:
|
||||
repo_name = project
|
||||
has_issues = 'has-issues' in options or default_has_issues
|
||||
has_downloads = 'has-downloads' in options or default_has_downloads
|
||||
has_wiki = 'has-wiki' in options or default_has_wiki
|
||||
try:
|
||||
org = orgs_dict[project_split[0].lower()]
|
||||
except KeyError:
|
||||
# We do not have control of this github org ignore the project.
|
||||
continue
|
||||
try:
|
||||
repo = org.get_repo(repo_name)
|
||||
except github.GithubException:
|
||||
repo = org.create_repo(repo_name,
|
||||
homepage=homepage,
|
||||
has_issues=has_issues,
|
||||
has_downloads=has_downloads,
|
||||
has_wiki=has_wiki)
|
||||
if description:
|
||||
repo.edit(repo_name, description=description)
|
||||
if homepage:
|
||||
repo.edit(repo_name, homepage=homepage)
|
||||
|
||||
repo.edit(repo_name, has_issues=has_issues,
|
||||
has_downloads=has_downloads,
|
||||
has_wiki=has_wiki)
|
||||
|
||||
if 'gerrit' not in [team.name for team in repo.get_teams()]:
|
||||
teams = org.get_teams()
|
||||
teams_dict = dict(zip([t.name.lower() for t in teams], teams))
|
||||
teams_dict['gerrit'].add_to_repos(repo)
|
||||
|
||||
remote_url = "ssh://localhost:29418/%s" % project
|
||||
if project not in project_list:
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
repo_path = os.path.join(tmpdir, 'repo')
|
||||
if upstream:
|
||||
run_command("git clone %(upstream)s %(repo_path)s" %
|
||||
dict(upstream=upstream, repo_path=repo_path))
|
||||
else:
|
||||
run_command("git init %s" % repo_path)
|
||||
with open(os.path.join(repo_path,
|
||||
".gitreview"), 'w') as gitreview:
|
||||
gitreview.write("""
|
||||
[gerrit]
|
||||
host=%s
|
||||
port=29418
|
||||
project=%s
|
||||
""" % (GERRIT_HOST, project_git))
|
||||
git_command(repo_path, "add .gitreview")
|
||||
cmd = "commit -a -m'Added .gitreview' --author=" \
|
||||
"'Openstack Project Creator " \
|
||||
"<openstack-infra@lists.openstack.org>'"
|
||||
git_command(repo_path, cmd)
|
||||
gerrit.createProject(project)
|
||||
|
||||
if not os.path.exists(project_dir):
|
||||
run_command("git --bare init %s" % project_dir)
|
||||
run_command("chown -R gerrit2:gerrit2 %s" % project_dir)
|
||||
|
||||
git_command(repo_path,
|
||||
"push --all %s" % remote_url,
|
||||
env=ssh_env)
|
||||
git_command(repo_path,
|
||||
"push --tags %s" % remote_url, env=ssh_env)
|
||||
finally:
|
||||
run_command("rm -fr %s" % tmpdir)
|
||||
|
||||
if 'acl_config' in section:
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
repo_path = os.path.join(tmpdir, 'repo')
|
||||
ret, _ = run_command_status("git init %s" % repo_path)
|
||||
if ret != 0:
|
||||
continue
|
||||
if (fetch_config(project, remote_url, repo_path, ssh_env) and
|
||||
copy_acl_config(project, repo_path,
|
||||
section['acl_config']) and
|
||||
create_groups_file(project, gerrit, repo_path)):
|
||||
push_acl_config(project, remote_url, repo_path, ssh_env)
|
||||
finally:
|
||||
run_command("rm -fr %s" % tmpdir)
|
||||
finally:
|
||||
os.unlink(ssh_env['GIT_SSH'])
|
||||
@ -1,92 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2012 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is designed to be called by a gerrit hook. It searched new
|
||||
# patchsets for strings like "bug FOO" and updates corresponding Launchpad
|
||||
# bugs status.
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import smtplib
|
||||
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
BASE_DIR = '/home/gerrit2/review_site'
|
||||
EMAIL_TEMPLATE = """
|
||||
Hi, I'd like you to take a look at this patch for potential
|
||||
%s.
|
||||
%s
|
||||
|
||||
Log:
|
||||
%s
|
||||
"""
|
||||
|
||||
def process_impact(git_log, args):
|
||||
"""Notify mail list of impact"""
|
||||
email_content = EMAIL_TEMPLATE % (args.impact, args.change_url, git_log)
|
||||
msg = MIMEText(email_content)
|
||||
msg['Subject'] = '[%s] %s review request change %s' % \
|
||||
(args.project, args.impact, args.change)
|
||||
msg['From'] = 'gerrit2@review.openstack.org'
|
||||
msg['To'] = args.dest_address
|
||||
|
||||
s = smtplib.SMTP('localhost')
|
||||
s.sendmail('gerrit2@review.openstack.org',
|
||||
args.dest_address, msg.as_string())
|
||||
s.quit()
|
||||
|
||||
def impacted(git_log, impact_string):
|
||||
"""Determine if a changes log indicates there is an impact"""
|
||||
return re.search(impact_string, git_log, re.IGNORECASE)
|
||||
|
||||
def extract_git_log(args):
|
||||
"""Extract git log of all merged commits"""
|
||||
cmd = ['git',
|
||||
'--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
|
||||
'log', '--no-merges', args.commit + '^1..' + args.commit]
|
||||
return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('hook')
|
||||
#common
|
||||
parser.add_argument('--change', default=None)
|
||||
parser.add_argument('--change-url', default=None)
|
||||
parser.add_argument('--project', default=None)
|
||||
parser.add_argument('--branch', default=None)
|
||||
parser.add_argument('--commit', default=None)
|
||||
#change-merged
|
||||
parser.add_argument('--submitter', default=None)
|
||||
#patchset-created
|
||||
parser.add_argument('--uploader', default=None)
|
||||
parser.add_argument('--patchset', default=None)
|
||||
# Not passed by gerrit:
|
||||
parser.add_argument('--impact', default=None)
|
||||
parser.add_argument('--dest-address', default=None)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get git log
|
||||
git_log = extract_git_log(args)
|
||||
|
||||
# Process impacts found in git log
|
||||
if impacted(git_log, args.impact):
|
||||
process_impact(git_log, args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,134 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is designed to be called by a gerrit hook. It searched new
|
||||
# patchsets for strings like "blueprint FOO" or "bp FOO" and updates
|
||||
# corresponding Launchpad blueprints with links back to the change.
|
||||
|
||||
from launchpadlib.launchpad import Launchpad
|
||||
from launchpadlib.uris import LPNET_SERVICE_ROOT
|
||||
import os
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import StringIO
|
||||
import ConfigParser
|
||||
import MySQLdb
|
||||
|
||||
BASE_DIR = '/home/gerrit2/review_site'
|
||||
GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
|
||||
'~/.launchpadlib/cache'))
|
||||
GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
|
||||
'~/.launchpadlib/creds'))
|
||||
GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG',
|
||||
'/home/gerrit2/review_site/etc/gerrit.config')
|
||||
GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG',
|
||||
'/home/gerrit2/review_site/etc/secure.config')
|
||||
SPEC_RE = re.compile(r'(blueprint|bp)\s*[#:]?\s*(\S+)', re.I)
|
||||
BODY_RE = re.compile(r'^\s+.*$')
|
||||
|
||||
def get_broken_config(filename):
|
||||
""" gerrit config ini files are broken and have leading tabs """
|
||||
text = ""
|
||||
with open(filename,"r") as conf:
|
||||
for line in conf.readlines():
|
||||
text = "%s%s" % (text, line.lstrip())
|
||||
|
||||
fp = StringIO.StringIO(text)
|
||||
c=ConfigParser.ConfigParser()
|
||||
c.readfp(fp)
|
||||
return c
|
||||
|
||||
GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG)
|
||||
SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG)
|
||||
DB_USER = GERRIT_CONFIG.get("database", "username")
|
||||
DB_PASS = SECURE_CONFIG.get("database","password")
|
||||
DB_DB = GERRIT_CONFIG.get("database","database")
|
||||
|
||||
def update_spec(launchpad, project, name, subject, link, topic=None):
|
||||
group, project = project.split('/')
|
||||
spec = launchpad.projects[project].getSpecification(name=name)
|
||||
if not spec: return
|
||||
|
||||
if spec.whiteboard:
|
||||
wb = spec.whiteboard.strip()
|
||||
else:
|
||||
wb = ''
|
||||
changed = False
|
||||
if topic:
|
||||
topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/',8)],
|
||||
topic)
|
||||
if topiclink not in wb:
|
||||
wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink)
|
||||
changed = True
|
||||
|
||||
if link not in wb:
|
||||
wb += "\n\n\nAddressed by: %(link)s\n %(subject)s\n" % dict(subject=subject,
|
||||
link=link)
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
spec.whiteboard = wb
|
||||
spec.lp_save()
|
||||
|
||||
def find_specs(launchpad, dbconn, args):
|
||||
git_log = subprocess.Popen(['git',
|
||||
'--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
|
||||
'log', '--no-merges',
|
||||
args.commit + '^1..' + args.commit],
|
||||
stdout=subprocess.PIPE).communicate()[0]
|
||||
|
||||
cur = dbconn.cursor()
|
||||
cur.execute("select subject, topic from changes where change_key=%s", args.change)
|
||||
subject, topic = cur.fetchone()
|
||||
specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)])
|
||||
|
||||
if topic:
|
||||
topicspec = topic.split('/')[-1]
|
||||
specs |= set([topicspec])
|
||||
|
||||
for spec in specs:
|
||||
update_spec(launchpad, args.project, spec, subject,
|
||||
args.change_url, topic)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('hook')
|
||||
#common
|
||||
parser.add_argument('--change', default=None)
|
||||
parser.add_argument('--change-url', default=None)
|
||||
parser.add_argument('--project', default=None)
|
||||
parser.add_argument('--branch', default=None)
|
||||
parser.add_argument('--commit', default=None)
|
||||
#change-merged
|
||||
parser.add_argument('--submitter', default=None)
|
||||
# patchset-created
|
||||
parser.add_argument('--uploader', default=None)
|
||||
parser.add_argument('--patchset', default=None)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
|
||||
GERRIT_CACHE_DIR,
|
||||
credentials_file = GERRIT_CREDENTIALS,
|
||||
version='devel')
|
||||
|
||||
conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB)
|
||||
|
||||
find_specs(launchpad, conn, args)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,244 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2011 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# This is designed to be called by a gerrit hook. It searched new
|
||||
# patchsets for strings like "bug FOO" and updates corresponding Launchpad
|
||||
# bugs status.
|
||||
|
||||
from launchpadlib.launchpad import Launchpad
|
||||
from launchpadlib.uris import LPNET_SERVICE_ROOT
|
||||
import os
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
|
||||
BASE_DIR = '/home/gerrit2/review_site'
|
||||
GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR',
|
||||
'~/.launchpadlib/cache'))
|
||||
GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS',
|
||||
'~/.launchpadlib/creds'))
|
||||
|
||||
|
||||
def add_change_proposed_message(bugtask, change_url, project, branch):
|
||||
subject = 'Fix proposed to %s (%s)' % (short_project(project), branch)
|
||||
body = 'Fix proposed to branch: %s\nReview: %s' % (branch, change_url)
|
||||
bugtask.bug.newMessage(subject=subject, content=body)
|
||||
|
||||
|
||||
def add_change_merged_message(bugtask, change_url, project, commit,
|
||||
submitter, branch, git_log):
|
||||
subject = 'Fix merged to %s (%s)' % (short_project(project), branch)
|
||||
git_url = 'http://github.com/%s/commit/%s' % (project, commit)
|
||||
body = '''Reviewed: %s
|
||||
Committed: %s
|
||||
Submitter: %s
|
||||
Branch: %s\n''' % (change_url, git_url, submitter, branch)
|
||||
body = body + '\n' + git_log
|
||||
bugtask.bug.newMessage(subject=subject, content=body)
|
||||
|
||||
|
||||
def set_in_progress(bugtask, launchpad, uploader, change_url):
|
||||
"""Set bug In progress with assignee being the uploader"""
|
||||
|
||||
# Retrieve uploader from Launchpad. Use email as search key if
|
||||
# provided, and only set if there is a clear match.
|
||||
try:
|
||||
searchkey = uploader[uploader.rindex("(") + 1:-1]
|
||||
except ValueError:
|
||||
searchkey = uploader
|
||||
persons = launchpad.people.findPerson(text=searchkey)
|
||||
if len(persons) == 1:
|
||||
bugtask.assignee = persons[0]
|
||||
|
||||
bugtask.status = "In Progress"
|
||||
bugtask.lp_save()
|
||||
|
||||
|
||||
def set_fix_committed(bugtask):
|
||||
"""Set bug fix committed"""
|
||||
|
||||
bugtask.status = "Fix Committed"
|
||||
bugtask.lp_save()
|
||||
|
||||
|
||||
def set_fix_released(bugtask):
|
||||
"""Set bug fix released"""
|
||||
|
||||
bugtask.status = "Fix Released"
|
||||
bugtask.lp_save()
|
||||
|
||||
|
||||
def release_fixcommitted(bugtask):
|
||||
"""Set bug FixReleased if it was FixCommitted"""
|
||||
|
||||
if bugtask.status == u'Fix Committed':
|
||||
set_fix_released(bugtask)
|
||||
|
||||
|
||||
def tag_in_branchname(bugtask, branch):
|
||||
"""Tag bug with in-branch-name tag (if name is appropriate)"""
|
||||
|
||||
lp_bug = bugtask.bug
|
||||
branch_name = branch.replace('/', '-')
|
||||
if branch_name.replace('-', '').isalnum():
|
||||
lp_bug.tags = lp_bug.tags + ["in-%s" % branch_name]
|
||||
lp_bug.tags.append("in-%s" % branch_name)
|
||||
lp_bug.lp_save()
|
||||
|
||||
|
||||
def short_project(full_project_name):
|
||||
"""Return the project part of the git repository name"""
|
||||
return full_project_name.split('/')[-1]
|
||||
|
||||
|
||||
def git2lp(full_project_name):
|
||||
"""Convert Git repo name to Launchpad project"""
|
||||
project_map = {
|
||||
'openstack/openstack-ci-puppet': 'openstack-ci',
|
||||
'openstack-ci/devstack-gate': 'openstack-ci',
|
||||
'openstack-ci/gerrit': 'openstack-ci',
|
||||
'openstack-ci/lodgeit': 'openstack-ci',
|
||||
'openstack-ci/meetbot': 'openstack-ci',
|
||||
}
|
||||
return project_map.get(full_project_name, short_project(full_project_name))
|
||||
|
||||
|
||||
def is_direct_release(full_project_name):
|
||||
"""Test against a list of projects who directly release changes."""
|
||||
return full_project_name in [
|
||||
'openstack-ci/devstack-gate',
|
||||
'openstack-ci/lodgeit',
|
||||
'openstack-ci/meetbot',
|
||||
'openstack-dev/devstack',
|
||||
'openstack/openstack-ci',
|
||||
'openstack/openstack-ci-puppet',
|
||||
'openstack/openstack-manuals',
|
||||
'openstack/tempest',
|
||||
]
|
||||
|
||||
|
||||
def process_bugtask(launchpad, bugtask, git_log, args):
|
||||
"""Apply changes to bugtask, based on hook / branch..."""
|
||||
|
||||
if args.hook == "change-merged":
|
||||
if args.branch == 'master':
|
||||
if is_direct_release(args.project):
|
||||
set_fix_released(bugtask)
|
||||
else:
|
||||
if bugtask.status != u'Fix Released':
|
||||
set_fix_committed(bugtask)
|
||||
elif args.branch == 'milestone-proposed':
|
||||
release_fixcommitted(bugtask)
|
||||
elif args.branch.startswith('stable/'):
|
||||
series = args.branch[7:]
|
||||
# Look for a related task matching the series
|
||||
for reltask in bugtask.related_tasks:
|
||||
if (reltask.bug_target_name.endswith("/" + series) and
|
||||
reltask.status != u'Fix Released'):
|
||||
# Use fixcommitted if there is any
|
||||
set_fix_committed(reltask)
|
||||
break
|
||||
else:
|
||||
# Use tagging if there isn't any
|
||||
tag_in_branchname(bugtask, args.branch)
|
||||
|
||||
add_change_merged_message(bugtask, args.change_url, args.project,
|
||||
args.commit, args.submitter, args.branch,
|
||||
git_log)
|
||||
|
||||
if args.hook == "patchset-created":
|
||||
if args.branch == 'master':
|
||||
if bugtask.status not in [u'Fix Committed', u'Fix Released']:
|
||||
set_in_progress(bugtask, launchpad, args.uploader,
|
||||
args.change_url)
|
||||
elif args.branch.startswith('stable/'):
|
||||
series = args.branch[7:]
|
||||
for reltask in bugtask.related_tasks:
|
||||
if (reltask.bug_target_name.endswith("/" + series) and
|
||||
reltask.status not in [u'Fix Committed', u'Fix Released']):
|
||||
set_in_progress(reltask, launchpad,
|
||||
args.uploader, args.change_url)
|
||||
break
|
||||
|
||||
if args.patchset == '1':
|
||||
add_change_proposed_message(bugtask, args.change_url,
|
||||
args.project, args.branch)
|
||||
|
||||
|
||||
def find_bugs(launchpad, git_log, args):
|
||||
"""Find bugs referenced in the git log and return related bugtasks"""
|
||||
|
||||
bug_regexp = r'([Bb]ug|[Ll][Pp])[\s#:]*(\d+)'
|
||||
tokens = re.split(bug_regexp, git_log)
|
||||
|
||||
# Extract unique bug tasks
|
||||
bugtasks = {}
|
||||
for token in tokens:
|
||||
if re.match('^\d+$', token) and (token not in bugtasks):
|
||||
try:
|
||||
lp_bug = launchpad.bugs[token]
|
||||
for lp_task in lp_bug.bug_tasks:
|
||||
if lp_task.bug_target_name == git2lp(args.project):
|
||||
bugtasks[token] = lp_task
|
||||
break
|
||||
except KeyError:
|
||||
# Unknown bug
|
||||
pass
|
||||
|
||||
return bugtasks.values()
|
||||
|
||||
|
||||
def extract_git_log(args):
|
||||
"""Extract git log of all merged commits"""
|
||||
cmd = ['git',
|
||||
'--git-dir=' + BASE_DIR + '/git/' + args.project + '.git',
|
||||
'log', '--no-merges', args.commit + '^1..' + args.commit]
|
||||
return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('hook')
|
||||
#common
|
||||
parser.add_argument('--change', default=None)
|
||||
parser.add_argument('--change-url', default=None)
|
||||
parser.add_argument('--project', default=None)
|
||||
parser.add_argument('--branch', default=None)
|
||||
parser.add_argument('--commit', default=None)
|
||||
#change-merged
|
||||
parser.add_argument('--submitter', default=None)
|
||||
#patchset-created
|
||||
parser.add_argument('--uploader', default=None)
|
||||
parser.add_argument('--patchset', default=None)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Connect to Launchpad
|
||||
launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT,
|
||||
GERRIT_CACHE_DIR,
|
||||
credentials_file=GERRIT_CREDENTIALS,
|
||||
version='devel')
|
||||
|
||||
# Get git log
|
||||
git_log = extract_git_log(args)
|
||||
|
||||
# Process bugtasks found in git log
|
||||
for bugtask in find_bugs(launchpad, git_log, args):
|
||||
process_bugtask(launchpad, bugtask, git_log, args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,72 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
# Copyright (C) 2011 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Add launchpad ids listed in the wiki CLA page to the CLA group in LP.
|
||||
|
||||
import os
|
||||
import urllib
|
||||
import re
|
||||
|
||||
from launchpadlib.launchpad import Launchpad
|
||||
from launchpadlib.uris import LPNET_SERVICE_ROOT
|
||||
|
||||
DEBUG = False
|
||||
|
||||
LP_CACHE_DIR = '~/.launchpadlib/cache'
|
||||
LP_CREDENTIALS = '~/.launchpadlib/creds'
|
||||
CONTRIBUTOR_RE = re.compile(r'.*?\|\|\s*(?P<name>.*?)\s*\|\|\s*(?P<login>.*?)\s*\|\|\s*(?P<trans>.*?)\s*\|\|.*?')
|
||||
LINK_RE = re.compile(r'\[\[.*\|\s*(?P<name>.*)\s*\]\]')
|
||||
|
||||
for check_path in (os.path.dirname(LP_CACHE_DIR),
|
||||
os.path.dirname(LP_CREDENTIALS)):
|
||||
if not os.path.exists(check_path):
|
||||
os.makedirs(check_path)
|
||||
|
||||
wiki_members = []
|
||||
for line in urllib.urlopen('http://wiki.openstack.org/Contributors?action=raw'):
|
||||
m = CONTRIBUTOR_RE.match(line)
|
||||
if m and m.group('login') and m.group('trans'):
|
||||
login = m.group('login')
|
||||
if login=="<#c0c0c0>'''Launchpad ID'''": continue
|
||||
l = LINK_RE.match(login)
|
||||
if l:
|
||||
login = l.group('name')
|
||||
wiki_members.append(login)
|
||||
|
||||
launchpad = Launchpad.login_with('CLA Team Sync', LPNET_SERVICE_ROOT,
|
||||
LP_CACHE_DIR,
|
||||
credentials_file = LP_CREDENTIALS,
|
||||
version='devel')
|
||||
|
||||
lp_members = []
|
||||
|
||||
team = launchpad.people['openstack-cla']
|
||||
for detail in team.members_details:
|
||||
user = None
|
||||
# detail.self_link ==
|
||||
# 'https://api.launchpad.net/1.0/~team/+member/${username}'
|
||||
login = detail.self_link.split('/')[-1]
|
||||
status = detail.status
|
||||
lp_members.append(login)
|
||||
|
||||
for wm in wiki_members:
|
||||
if wm not in lp_members:
|
||||
print "Need to add %s to LP" % (wm)
|
||||
try:
|
||||
person = launchpad.people[wm]
|
||||
except:
|
||||
print 'Unable to find %s on LP'%wm
|
||||
continue
|
||||
status = team.addMember(person=person, status="Approved")
|
||||
@ -9,8 +9,8 @@ class gerrit::cron(
|
||||
user => 'gerrit2',
|
||||
hour => '6',
|
||||
minute => '3',
|
||||
command => "python /usr/local/gerrit/scripts/expire_old_reviews.py ${script_user} ${script_key_file}",
|
||||
require => File['/usr/local/gerrit/scripts'],
|
||||
command => "python /usr/local/bin/expire-old-reviews ${script_user} ${script_key_file}",
|
||||
require => Class['jeepyb'],
|
||||
}
|
||||
|
||||
cron { 'gerrit_repack':
|
||||
|
||||
@ -111,6 +111,7 @@ class gerrit(
|
||||
$testmode = false
|
||||
) {
|
||||
include apache
|
||||
include jeepyb
|
||||
include pip
|
||||
|
||||
$java_home = $::lsbdistcodename ? {
|
||||
@ -490,13 +491,7 @@ class gerrit(
|
||||
}
|
||||
|
||||
file { '/usr/local/gerrit/scripts':
|
||||
ensure => directory,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
recurse => true,
|
||||
require => File['/usr/local/gerrit'],
|
||||
source => 'puppet:///modules/gerrit/scripts',
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
# Install Bouncy Castle's OpenPGP plugin and populate the contact store
|
||||
|
||||
@ -5,8 +5,8 @@ class gerrit::remotes($ensure=present) {
|
||||
ensure => $ensure,
|
||||
user => 'gerrit2',
|
||||
minute => '*/30',
|
||||
command => 'sleep $((RANDOM\%60+90)) && python /usr/local/gerrit/scripts/fetch_remotes.py',
|
||||
require => File['/usr/local/gerrit/scripts'],
|
||||
command => 'sleep $((RANDOM\%60+90)) && /usr/local/bin/fetch-remotes',
|
||||
require => Class['jeepyb'],
|
||||
}
|
||||
|
||||
file { '/home/gerrit2/remotes.config':
|
||||
|
||||
@ -1,94 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
# Copyright (C) 2011 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Github pull requests closer reads a project config file called projects.yaml
|
||||
# It should look like:
|
||||
|
||||
# - homepage: http://openstack.org
|
||||
# team-id: 153703
|
||||
# has-wiki: False
|
||||
# has-issues: False
|
||||
# has-downloads: False
|
||||
# ---
|
||||
# - project: PROJECT_NAME
|
||||
# options:
|
||||
# - has-pull-requests
|
||||
|
||||
# Github authentication information is read from github.secure.config,
|
||||
# which should look like:
|
||||
|
||||
# [github]
|
||||
# username = GITHUB_USERNAME
|
||||
# password = GITHUB_PASSWORD
|
||||
#
|
||||
# or
|
||||
#
|
||||
# [github]
|
||||
# oauth_token = GITHUB_OAUTH_TOKEN
|
||||
|
||||
import ConfigParser
|
||||
import github
|
||||
import os
|
||||
import yaml
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.ERROR)
|
||||
|
||||
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
|
||||
'/home/gerrit2/projects.yaml')
|
||||
GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
|
||||
'/etc/github/github.secure.config')
|
||||
|
||||
MESSAGE = """Thank you for contributing to OpenStack!
|
||||
|
||||
%(project)s uses Gerrit for code review.
|
||||
|
||||
Please visit http://wiki.openstack.org/GerritWorkflow and follow the instructions there to upload your change to Gerrit.
|
||||
"""
|
||||
|
||||
secure_config = ConfigParser.ConfigParser()
|
||||
secure_config.read(GITHUB_SECURE_CONFIG)
|
||||
(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))]
|
||||
|
||||
if secure_config.has_option("github", "oauth_token"):
|
||||
ghub = github.Github(secure_config.get("github", "oauth_token"))
|
||||
else:
|
||||
ghub = github.Github(secure_config.get("github", "username"),
|
||||
secure_config.get("github", "password"))
|
||||
|
||||
orgs = ghub.get_user().get_orgs()
|
||||
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
|
||||
for section in config:
|
||||
project = section['project']
|
||||
|
||||
# Make sure we're supposed to close pull requests for this project:
|
||||
if 'options' in section and 'has-pull-requests' in section['options']:
|
||||
continue
|
||||
|
||||
# Find the project's repo
|
||||
project_split = project.split('/', 1)
|
||||
if len(project_split) > 1:
|
||||
repo = orgs_dict[project_split[0].lower()].get_repo(project_split[1])
|
||||
else:
|
||||
repo = ghub.get_user().get_repo(project)
|
||||
|
||||
# Close each pull request
|
||||
pull_requests = repo.get_pulls("open")
|
||||
for req in pull_requests:
|
||||
vars = dict(project=project)
|
||||
issue_data = {"url": repo.url + "/issues/" + str(req.number)}
|
||||
issue = github.Issue.Issue(req._requester, issue_data, completed = True)
|
||||
issue.create_comment(MESSAGE % vars)
|
||||
req.edit(state = "closed")
|
||||
@ -7,6 +7,7 @@ class github(
|
||||
$project_password = '',
|
||||
$projects = []
|
||||
) {
|
||||
include jeepyb
|
||||
include pip
|
||||
|
||||
package { 'PyGithub':
|
||||
@ -80,20 +81,14 @@ class github(
|
||||
}
|
||||
|
||||
file { '/usr/local/github/scripts':
|
||||
ensure => directory,
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
owner => 'root',
|
||||
recurse => true,
|
||||
require => File['/usr/local/github'],
|
||||
source => 'puppet:///modules/github/scripts',
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
cron { 'githubclosepull':
|
||||
command => 'sleep $((RANDOM\%60+90)) && python /usr/local/github/scripts/close_pull_requests.py',
|
||||
command => 'sleep $((RANDOM\%60+90)) && /usr/local/bin/close-pull-requests',
|
||||
minute => '*/5',
|
||||
require => [
|
||||
File['/usr/local/github/scripts'],
|
||||
Class['jeepyb'],
|
||||
Package['python-yaml'],
|
||||
Package['PyGithub'],
|
||||
],
|
||||
|
||||
54
modules/jeepyb/manifests/init.pp
Normal file
54
modules/jeepyb/manifests/init.pp
Normal file
@ -0,0 +1,54 @@
|
||||
# == Class: jeepyb
|
||||
#
|
||||
class jeepyb (
|
||||
$git_source_repo = 'https://github.com/openstack-ci/jeepyb.git',
|
||||
) {
|
||||
$packages = [
|
||||
'python-mysqldb',
|
||||
'python-paramiko',
|
||||
]
|
||||
|
||||
package { $packages:
|
||||
ensure => present,
|
||||
}
|
||||
|
||||
if ! defined(Package['Pygithub']) {
|
||||
package { 'PyGithub':
|
||||
ensure => latest,
|
||||
provider => pip,
|
||||
require => Class['pip'],
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(Package['gerritlib']) {
|
||||
package { 'gerritlib':
|
||||
ensure => latest,
|
||||
provider => pip,
|
||||
require => Class['pip'],
|
||||
}
|
||||
}
|
||||
|
||||
# A lot of things need yaml, be conservative requiring this package to avoid
|
||||
# conflicts with other modules.
|
||||
if ! defined(Package['python-yaml']) {
|
||||
package { 'python-yaml':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
vcsrepo { '/opt/jeepyb':
|
||||
ensure => latest,
|
||||
provider => git,
|
||||
revision => 'master',
|
||||
source => $git_source_repo,
|
||||
}
|
||||
|
||||
exec { 'install_jeepyb' :
|
||||
command => 'python setup.py install',
|
||||
cwd => '/opt/jeepyb',
|
||||
path => '/bin:/usr/bin',
|
||||
refreshonly => true,
|
||||
subscribe => Vcsrepo['/opt/jeepyb'],
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Use timeout to kill any process running longer than 10 minutes.
|
||||
timeout -k 2m 10m python /usr/local/gerrit/scripts/update_bug.py change-merged "$@"
|
||||
timeout -k 2m 10m /usr/local/bin/update-bug change-merged "$@"
|
||||
|
||||
@ -1,265 +0,0 @@
|
||||
#!/usr/bin/env python2.6
|
||||
|
||||
# Copyright (c) 2010, Code Aurora Forum. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met:
|
||||
# # Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# # Redistributions in binary form must reproduce the above
|
||||
# copyright notice, this list of conditions and the following
|
||||
# disclaimer in the documentation and/or other materials provided
|
||||
# with the distribution.
|
||||
# # Neither the name of Code Aurora Forum, Inc. nor the names of its
|
||||
# contributors may be used to endorse or promote products derived
|
||||
# from this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
|
||||
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
|
||||
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
|
||||
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
|
||||
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# This script is designed to detect when a patchset uploaded to Gerrit is
|
||||
# 'identical' (determined via git-patch-id) and reapply reviews onto the new
|
||||
# patchset from the previous patchset.
|
||||
|
||||
# Get usage and help info by running: ./trivial_rebase.py --help
|
||||
# Documentation is available here: https://www.codeaurora.org/xwiki/bin/QAEP/Gerrit
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from sys import exit
|
||||
|
||||
from optparse import OptionParser as _realOptionParser, AmbiguousOptionError, \
|
||||
BadOptionError
|
||||
class OptionParser(_realOptionParser):
|
||||
"""Make OptionParser silently swallow unrecognized options."""
|
||||
def _process_args(self, largs, rargs, values):
|
||||
while rargs:
|
||||
try:
|
||||
_realOptionParser._process_args(self, largs, rargs, values)
|
||||
except (AmbiguousOptionError, BadOptionError), e:
|
||||
largs.append(e.opt_str)
|
||||
|
||||
class CheckCallError(OSError):
|
||||
"""CheckCall() returned non-0."""
|
||||
def __init__(self, command, cwd, retcode, stdout, stderr=None):
|
||||
OSError.__init__(self, command, cwd, retcode, stdout, stderr)
|
||||
self.command = command
|
||||
self.cwd = cwd
|
||||
self.retcode = retcode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
def CheckCall(command, cwd=None):
|
||||
"""Like subprocess.check_call() but returns stdout.
|
||||
|
||||
Works on python 2.4
|
||||
"""
|
||||
try:
|
||||
process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE)
|
||||
std_out, std_err = process.communicate()
|
||||
except OSError, e:
|
||||
raise CheckCallError(command, cwd, e.errno, None)
|
||||
if process.returncode:
|
||||
raise CheckCallError(command, cwd, process.returncode, std_out, std_err)
|
||||
return std_out, std_err
|
||||
|
||||
def Gssh(options, api_command):
|
||||
"""Makes a Gerrit API call via SSH and returns the stdout results."""
|
||||
ssh_cmd = ['ssh',
|
||||
'-l', 'Gerrit Code Review',
|
||||
'-p', options.port,
|
||||
'-i', options.private_key_path,
|
||||
options.server,
|
||||
api_command]
|
||||
try:
|
||||
return CheckCall(ssh_cmd)[0]
|
||||
except CheckCallError, e:
|
||||
import sys
|
||||
err_template = "call: %s\nreturn code: %s\nstdout: %s\nstderr: %s\n"
|
||||
sys.stderr.write(err_template%(ssh_cmd, e.retcode, e.stdout, e.stderr))
|
||||
raise
|
||||
|
||||
def GsqlQuery(sql_query, options):
|
||||
"""Runs a gerrit gsql query and returns the result"""
|
||||
gsql_cmd = "gerrit gsql --format JSON -c %s"%sql_query
|
||||
gsql_out = Gssh(options, gsql_cmd)
|
||||
new_out = gsql_out.replace('}}\n', '}}\nsplit here\n')
|
||||
return new_out.split('split here\n')
|
||||
|
||||
def FindPrevRev(options):
|
||||
"""Finds the revision of the previous patch set on the change"""
|
||||
sql_query = ("\"SELECT revision FROM patch_sets,changes WHERE "
|
||||
"patch_sets.change_id = changes.change_id AND "
|
||||
"patch_sets.patch_set_id = %s AND "
|
||||
"changes.change_key = \'%s\'\"" % ((options.patchset - 1),
|
||||
options.changeId))
|
||||
revisions = GsqlQuery(sql_query, options)
|
||||
|
||||
json_dict = json.loads(revisions[0], strict=False)
|
||||
return json_dict["columns"]["revision"]
|
||||
|
||||
def GetApprovals(options):
|
||||
"""Get all the approvals on a specific patch set
|
||||
|
||||
Returns a list of approval dicts"""
|
||||
sql_query = ("\"SELECT value,account_id,category_id FROM patch_set_approvals "
|
||||
"WHERE patch_set_id = %s AND change_id = (SELECT change_id FROM "
|
||||
"changes WHERE change_key = \'%s\') AND value <> 0\""
|
||||
% ((options.patchset - 1), options.changeId))
|
||||
gsql_out = GsqlQuery(sql_query, options)
|
||||
approvals = []
|
||||
for json_str in gsql_out:
|
||||
dict = json.loads(json_str, strict=False)
|
||||
if dict["type"] == "row":
|
||||
approvals.append(dict["columns"])
|
||||
return approvals
|
||||
|
||||
def GetPatchId(revision, consider_whitespace=False):
|
||||
git_show_cmd = ['git', 'show', revision]
|
||||
patch_id_cmd = ['git', 'patch-id']
|
||||
patch_id_process = subprocess.Popen(patch_id_cmd, stdout=subprocess.PIPE,
|
||||
stdin=subprocess.PIPE)
|
||||
git_show_process = subprocess.Popen(git_show_cmd, stdout=subprocess.PIPE)
|
||||
if consider_whitespace:
|
||||
# This matches on change lines in the patch (those starting with "+"
|
||||
# or "-" but not followed by another of the same), then replaces any
|
||||
# space or tab characters with "%" before calculating a patch-id.
|
||||
replace_ws_cmd = ['sed', r'/^\(+[^+]\|-[^-]\)/y/ \t/%%/']
|
||||
replace_ws_process = subprocess.Popen(replace_ws_cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stdin=subprocess.PIPE)
|
||||
return patch_id_process.communicate(
|
||||
replace_ws_process.communicate(git_show_process.communicate()[0])[0]
|
||||
)[0]
|
||||
else:
|
||||
return patch_id_process.communicate(git_show_process.communicate()[0])[0]
|
||||
|
||||
def SuExec(options, as_user, cmd):
|
||||
suexec_cmd = "suexec --as %s -- %s"%(as_user, cmd)
|
||||
Gssh(options, suexec_cmd)
|
||||
|
||||
def DiffCommitMessages(commit1, commit2):
|
||||
log_cmd1 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"',
|
||||
commit1 + '^!']
|
||||
commit1_log = CheckCall(log_cmd1)
|
||||
log_cmd2 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"',
|
||||
commit2 + '^!']
|
||||
commit2_log = CheckCall(log_cmd2)
|
||||
if commit1_log != commit2_log:
|
||||
return True
|
||||
return False
|
||||
|
||||
def Main():
|
||||
usage = "usage: %prog <required options> [optional options]"
|
||||
parser = OptionParser(usage=usage)
|
||||
parser.add_option("--change", dest="changeId", help="Change identifier")
|
||||
parser.add_option("--project", help="Project path in Gerrit")
|
||||
parser.add_option("--commit", help="Git commit-ish for this patchset")
|
||||
parser.add_option("--patchset", type="int", help="The patchset number")
|
||||
parser.add_option("--role-user", dest="role_user",
|
||||
help="E-mail/ID of user commenting on commit messages")
|
||||
parser.add_option("--private-key-path", dest="private_key_path",
|
||||
help="Full path to Gerrit SSH daemon's private host key")
|
||||
parser.add_option("--server-port", dest="port", default='29418',
|
||||
help="Port to connect to Gerrit's SSH daemon "
|
||||
"[default: %default]")
|
||||
parser.add_option("--server", dest="server", default="localhost",
|
||||
help="Server name/address for Gerrit's SSH daemon "
|
||||
"[default: %default]")
|
||||
parser.add_option("--whitespace", action="store_true",
|
||||
help="Treat whitespace as significant")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if not options.changeId:
|
||||
parser.print_help()
|
||||
exit(0)
|
||||
|
||||
if options.patchset == 1:
|
||||
# Nothing to detect on first patchset
|
||||
exit(0)
|
||||
prev_revision = None
|
||||
prev_revision = FindPrevRev(options)
|
||||
if not prev_revision:
|
||||
# Couldn't find a previous revision
|
||||
exit(0)
|
||||
prev_patch_id = GetPatchId(prev_revision)
|
||||
cur_patch_id = GetPatchId(options.commit)
|
||||
if cur_patch_id.split()[0] != prev_patch_id.split()[0]:
|
||||
# patch-ids don't match
|
||||
exit(0)
|
||||
# Patch ids match. This is a trivial rebase.
|
||||
# In addition to patch-id we should check if whitespace content changed. Some
|
||||
# languages are more sensitive to whitespace than others, and some changes
|
||||
# may either introduce or be intended to fix style problems specifically
|
||||
# involving whitespace as well.
|
||||
if options.whitespace:
|
||||
prev_patch_ws = GetPatchId(prev_revision, consider_whitespace=True)
|
||||
cur_patch_ws = GetPatchId(options.commit, consider_whitespace=True)
|
||||
if cur_patch_ws.split()[0] != prev_patch_ws.split()[0]:
|
||||
# Insert a comment into the change letting the approvers know only the
|
||||
# whitespace changed
|
||||
comment_msg = "\"New patchset patch-id matches previous patchset, " \
|
||||
"but whitespace content has changed.\""
|
||||
comment_cmd = ['gerrit', 'approve', '--project', options.project,
|
||||
'--message', comment_msg, options.commit]
|
||||
SuExec(options, options.role_user, ' '.join(comment_cmd))
|
||||
exit(0)
|
||||
|
||||
# We should also check if the commit message changed. Most approvers would
|
||||
# want to re-review changes when the commit message changes.
|
||||
changed = DiffCommitMessages(prev_revision, options.commit)
|
||||
if changed:
|
||||
# Insert a comment into the change letting the approvers know only the
|
||||
# commit message changed
|
||||
comment_msg = "\"New patchset patch-id matches previous patchset, " \
|
||||
"but commit message has changed.\""
|
||||
comment_cmd = ['gerrit', 'approve', '--project', options.project,
|
||||
'--message', comment_msg, options.commit]
|
||||
SuExec(options, options.role_user, ' '.join(comment_cmd))
|
||||
exit(0)
|
||||
|
||||
# Need to get all approvals on prior patch set, then suexec them onto
|
||||
# this patchset.
|
||||
approvals = GetApprovals(options)
|
||||
gerrit_approve_msg = ("\'Automatically re-added by Gerrit trivial rebase "
|
||||
"detection script.\'")
|
||||
for approval in approvals:
|
||||
# Note: Sites with different 'copy_min_score' values in the
|
||||
# approval_categories DB table might want different behavior here.
|
||||
# Additional categories should also be added if desired.
|
||||
if approval["category_id"] == "CRVW":
|
||||
approve_category = '--code-review'
|
||||
elif approval["category_id"] == "VRIF":
|
||||
# Don't re-add verifies
|
||||
#approve_category = '--verified'
|
||||
continue
|
||||
elif approval["category_id"] == "SUBM":
|
||||
# We don't care about previous submit attempts
|
||||
continue
|
||||
elif approval["category_id"] == "APRV":
|
||||
# Similarly squash old approvals
|
||||
continue
|
||||
else:
|
||||
print "Unsupported category: %s" % approval
|
||||
exit(0)
|
||||
|
||||
score = approval["value"]
|
||||
gerrit_approve_cmd = ['gerrit', 'approve', '--project', options.project,
|
||||
'--message', gerrit_approve_msg, approve_category,
|
||||
score, options.commit]
|
||||
SuExec(options, approval["account_id"], ' '.join(gerrit_approve_cmd))
|
||||
exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
Main()
|
||||
@ -214,17 +214,6 @@ class openstack_project::gerrit (
|
||||
require => Class['::gerrit'],
|
||||
}
|
||||
|
||||
file { '/usr/local/gerrit/scripts/trivial_rebase.py':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0444',
|
||||
source =>
|
||||
'puppet:///modules/openstack_project/gerrit/scripts/trivial_rebase.py',
|
||||
replace => true,
|
||||
require => Class['::gerrit'],
|
||||
}
|
||||
|
||||
if ($projects_file != 'UNDEF') {
|
||||
if ($replicate_local) {
|
||||
file { $local_git_dir:
|
||||
@ -264,7 +253,7 @@ class openstack_project::gerrit (
|
||||
}
|
||||
|
||||
exec { 'manage_projects':
|
||||
command => '/usr/local/gerrit/scripts/manage_projects.py',
|
||||
command => '/usr/local/bin/manage-projects',
|
||||
subscribe => [
|
||||
File['/home/gerrit2/projects.yaml'],
|
||||
File['/home/gerrit2/acls'],
|
||||
@ -273,6 +262,7 @@ class openstack_project::gerrit (
|
||||
require => [
|
||||
File['/home/gerrit2/projects.yaml'],
|
||||
File['/home/gerrit2/acls'],
|
||||
Class['jeepyb'],
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Use timeout to kill any process running longer than 10 minutes.
|
||||
timeout -k 2m 10m python /usr/local/gerrit/scripts/update_blueprint.py patchset-created "$@"
|
||||
timeout -k 2m 10m python /usr/local/gerrit/scripts/update_bug.py patchset-created "$@"
|
||||
timeout -k 2m 10m python /usr/local/gerrit/scripts/notify_impact.py patchset-created "$@" --impact DocImpact --dest-address 'openstack-docs@lists.openstack.org'
|
||||
timeout -k 2m 10m python /usr/local/gerrit/scripts/notify_impact.py patchset-created "$@" --impact SecurityImpact --dest-address 'openstack-ossg@lists.launchpad.net'
|
||||
timeout -k 2m 10m python /usr/local/gerrit/scripts/trivial_rebase.py \
|
||||
timeout -k 2m 10m /usr/local/bin/update-blueprint patchset-created "$@"
|
||||
timeout -k 2m 10m /usr/local/bin/update-bug patchset-created "$@"
|
||||
timeout -k 2m 10m /usr/local/bin/notify-impact patchset-created "$@" --impact DocImpact --dest-address 'openstack-docs@lists.openstack.org'
|
||||
timeout -k 2m 10m /usr/local/bin/notify-impact patchset-created "$@" --impact SecurityImpact --dest-address 'openstack-ossg@lists.launchpad.net'
|
||||
timeout -k 2m 10m /usr/local/bin/trivial-rebase \
|
||||
patchset-created \
|
||||
--whitespace \
|
||||
--private-key-path=<%= ssh_host_key %> \
|
||||
|
||||
@ -1,90 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import urllib
|
||||
import datetime
|
||||
import sys
|
||||
import re
|
||||
import md5
|
||||
|
||||
source_cache = sys.argv[1]
|
||||
destination_mirror = sys.argv[2]
|
||||
|
||||
PACKAGE_VERSION_RE = re.compile(r'(.*)-[0-9]')
|
||||
|
||||
packages = {}
|
||||
package_count = 0
|
||||
|
||||
for filename in os.listdir(source_cache):
|
||||
if filename.endswith('content-type'):
|
||||
continue
|
||||
|
||||
realname = urllib.unquote(filename)
|
||||
# The ? accounts for sourceforge downloads
|
||||
tarball = os.path.basename(realname).split("?")[0]
|
||||
name_match = PACKAGE_VERSION_RE.search(tarball)
|
||||
|
||||
if name_match is None:
|
||||
continue
|
||||
package_name = name_match.group(1)
|
||||
|
||||
version_list = packages.get(package_name,{})
|
||||
version_list[tarball] = filename
|
||||
packages[package_name] = version_list
|
||||
package_count = package_count + 1
|
||||
|
||||
full_html = open(os.path.join(destination_mirror, "full.html"), 'w')
|
||||
simple_html = open(os.path.join(destination_mirror, "index.html"), 'w')
|
||||
|
||||
header = "<html><head><title>PyPI Mirror</title></head><body><h1>PyPI Mirror</h1><h2>Last update: %s</h2>\n\n" % datetime.datetime.utcnow().strftime("%c UTC")
|
||||
full_html.write(header)
|
||||
simple_html.write(header)
|
||||
|
||||
for package_name, versions in packages.items():
|
||||
destination_dir = os.path.join(destination_mirror, package_name)
|
||||
if not os.path.isdir(destination_dir):
|
||||
os.makedirs(destination_dir)
|
||||
safe_dir = urllib.quote(package_name)
|
||||
simple_html.write("<a href='%s'>%s</a><br />\n" % (safe_dir, safe_dir))
|
||||
with open(os.path.join(destination_dir, "index.html"), 'w') as index:
|
||||
index.write("""<html><head>
|
||||
<title>%s – PyPI Mirror</title>
|
||||
</head><body>\n""" % package_name)
|
||||
for tarball, filename in versions.items():
|
||||
source_path = os.path.join(source_cache, filename)
|
||||
destination_path = os.path.join(destination_dir, tarball)
|
||||
with open(destination_path, 'w') as dest:
|
||||
src = open(source_path, 'r').read()
|
||||
md5sum = md5.md5(src).hexdigest()
|
||||
dest.write(src)
|
||||
|
||||
safe_name = urllib.quote(tarball)
|
||||
|
||||
full_html.write("<a href='%s/%s'>%s</a><br />\n" % (safe_dir,
|
||||
safe_name,
|
||||
safe_name))
|
||||
index.write("<a href='%s#md5=%s'>%s</a>\n" % (safe_name,
|
||||
md5sum,
|
||||
safe_name))
|
||||
index.write("</body></html>\n")
|
||||
footer = """<p class='footer'>Generated by process_cache.py; %d
|
||||
packages mirrored. </p>
|
||||
</body></html>\n""" % package_count
|
||||
full_html.write(footer)
|
||||
full_html.close()
|
||||
simple_html.write(footer)
|
||||
simple_html.close()
|
||||
@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
# This file is managed by puppet.
|
||||
# https://github.com/openstack/openstack-ci-puppet
|
||||
|
||||
export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE:-/var/cache/pip}
|
||||
export PIP_TEMP_DOWNLOAD=${PIP_TEMP_DOWNLOAD:-/var/lib/pip-download}
|
||||
|
||||
project=$1
|
||||
pip_command='/usr/local/bin/pip install -M -U -I --exists-action=w --no-install'
|
||||
|
||||
cd ${PIP_TEMP_DOWNLOAD}
|
||||
short_project=`echo ${project} | cut -f2 -d/`
|
||||
if [ ! -d ${short_project} ] ; then
|
||||
git clone git://github.com/${project}.git ${short_project} >/dev/null 2>&1
|
||||
fi
|
||||
cd ${short_project}
|
||||
$pip_command pip
|
||||
git fetch origin
|
||||
for branch in `git branch -a | grep remotes.origin | grep -v origin.HEAD | awk '{print $1}' ` ; do
|
||||
git reset --hard $branch
|
||||
git clean -x -f -d -q
|
||||
echo "*********************"
|
||||
echo "Fetching pip requires for $project:$branch"
|
||||
for requires_file in tools/pip-requires tools/test-requires ; do
|
||||
if [ -f ${requires_file} ] ; then
|
||||
$pip_command -r $requires_file
|
||||
fi
|
||||
done
|
||||
done
|
||||
@ -1,81 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
# Copyright (C) 2011 OpenStack, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# run_mirrors reads a project config file called projects.yaml
|
||||
# It should look like:
|
||||
|
||||
# - project: PROJECT_NAME
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import shlex
|
||||
import yaml
|
||||
|
||||
def run_command(cmd, status=False, env={}):
|
||||
cmd_list = shlex.split(str(cmd))
|
||||
newenv = os.environ
|
||||
newenv.update(env)
|
||||
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, env=newenv)
|
||||
(out, nothing) = p.communicate()
|
||||
if status:
|
||||
return (p.returncode, out.strip())
|
||||
return out.strip()
|
||||
|
||||
|
||||
def run_command_status(cmd, env={}):
|
||||
return run_command(cmd, True, env)
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.ERROR)
|
||||
|
||||
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
|
||||
'/etc/openstackci/projects.yaml')
|
||||
PIP_TEMP_DOWNLOAD = os.environ.get('PIP_TEMP_DOWNLOAD',
|
||||
'/var/lib/pip-download')
|
||||
GIT_SOURCE = os.environ.get('GIT_SOURCE', 'https://github.com')
|
||||
pip_command = '/usr/local/bin/pip install -M -U -I --exists-action=w ' \
|
||||
'--no-install %s'
|
||||
|
||||
run_command(pip_command % "pip")
|
||||
|
||||
(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))]
|
||||
|
||||
for section in config:
|
||||
project = section['project']
|
||||
|
||||
os.chdir(PIP_TEMP_DOWNLOAD)
|
||||
short_project = project.split('/')[1]
|
||||
if not os.path.isdir(short_project):
|
||||
run_command("git clone %s/%s.git %s" % (GIT_SOURCE, project,
|
||||
short_project))
|
||||
os.chdir(short_project)
|
||||
run_command("git fetch origin")
|
||||
|
||||
for branch in run_command("git branch -a").split("\n"):
|
||||
branch = branch.strip()
|
||||
if (not branch.startswith("remotes/origin")
|
||||
or "origin/HEAD" in branch):
|
||||
continue
|
||||
run_command("git reset --hard %s" % branch)
|
||||
run_command("git clean -x -f -d -q")
|
||||
print("*********************")
|
||||
print("Fetching pip requires for %s:%s" % (project, branch))
|
||||
for requires_file in ("tools/pip-requires", "tools/test-requires"):
|
||||
if os.path.exists(requires_file):
|
||||
stanza = "-r %s" % requires_file
|
||||
run_command(pip_command % stanza)
|
||||
|
||||
@ -15,6 +15,7 @@ class pypimirror(
|
||||
include apache
|
||||
include pip
|
||||
include remove_nginx
|
||||
include jeepyb
|
||||
|
||||
package { 'python-yaml':
|
||||
ensure => present,
|
||||
@ -68,30 +69,15 @@ class pypimirror(
|
||||
}
|
||||
|
||||
file { '/usr/local/mirror_scripts/run_mirror.py':
|
||||
ensure => present,
|
||||
mode => '0755',
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
source => 'puppet:///modules/pypimirror/run_mirror.py',
|
||||
require => File['/usr/local/mirror_scripts'],
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
file { '/usr/local/mirror_scripts/pull-repo.sh':
|
||||
ensure => present,
|
||||
mode => '0755',
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
source => 'puppet:///modules/pypimirror/pull-repo.sh',
|
||||
require => File['/usr/local/mirror_scripts'],
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
file { '/usr/local/mirror_scripts/process_cache.py':
|
||||
ensure => present,
|
||||
mode => '0755',
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
source => 'puppet:///modules/pypimirror/process_cache.py',
|
||||
require => File['/usr/local/mirror_scripts'],
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
# Add cron job to update the mirror
|
||||
|
||||
@ -6,5 +6,5 @@ export PIP_DOWNLOAD_CACHE=<%= pip_cache %>
|
||||
export PIP_TEMP_DOWNLOAD=<%= pip_download %>
|
||||
export MIRROR_FILE_PATH=<%= mirror_file_path %>
|
||||
export LOG_FILENAME=<%= log_filename %>
|
||||
python /usr/local/mirror_scripts/run_mirror.py <%= git_source %> >>$LOG_FILENAME
|
||||
python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}
|
||||
/usr/local/bin/run-mirror <%= git_source %> >>$LOG_FILENAME
|
||||
/usr/local/bin/process-cache ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user