From 52db16762bc1da7f6670d0f1449bf1f18933a11a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 22 Nov 2012 10:45:10 -0800 Subject: [PATCH] Consume jeepyb. Instead of keeping many of these files directly in the tree, use them from the out-of-tree jeepyb project, which makes them easier to consume for other people who are not us. Change-Id: Id704f2e17dd80709ef63cbbf2c5475a08a835f91 Reviewed-on: https://review.openstack.org/16777 Reviewed-by: Clark Boylan Reviewed-by: James E. Blair Approved: James E. Blair Tested-by: Jenkins --- doc/source/gerrit.rst | 2 +- .../files/scripts/expire_old_reviews.py | 78 ----- modules/gerrit/files/scripts/fetch_remotes.py | 76 ---- .../gerrit/files/scripts/get_group_uuid.py | 29 -- .../gerrit/files/scripts/manage_projects.py | 327 ------------------ modules/gerrit/files/scripts/notify_impact.py | 92 ----- .../gerrit/files/scripts/update_blueprint.py | 134 ------- modules/gerrit/files/scripts/update_bug.py | 244 ------------- .../gerrit/files/scripts/update_cla_group.py | 72 ---- modules/gerrit/manifests/cron.pp | 4 +- modules/gerrit/manifests/init.pp | 9 +- modules/gerrit/manifests/remotes.pp | 4 +- .../files/scripts/close_pull_requests.py | 94 ----- modules/github/manifests/init.pp | 13 +- modules/jeepyb/manifests/init.pp | 54 +++ .../files/gerrit/change-merged | 2 +- .../files/gerrit/scripts/trivial_rebase.py | 265 -------------- modules/openstack_project/manifests/gerrit.pp | 14 +- .../templates/gerrit_patchset-created.erb | 10 +- modules/pypimirror/files/process_cache.py | 90 ----- modules/pypimirror/files/pull-repo.sh | 29 -- modules/pypimirror/files/run_mirror.py | 81 ----- modules/pypimirror/manifests/init.pp | 22 +- .../pypimirror/templates/run-mirror.sh.erb | 4 +- 24 files changed, 79 insertions(+), 1670 deletions(-) delete mode 100644 modules/gerrit/files/scripts/expire_old_reviews.py delete mode 100755 modules/gerrit/files/scripts/fetch_remotes.py delete mode 100644 modules/gerrit/files/scripts/get_group_uuid.py delete mode 100755 modules/gerrit/files/scripts/manage_projects.py delete mode 100755 modules/gerrit/files/scripts/notify_impact.py delete mode 100755 modules/gerrit/files/scripts/update_blueprint.py delete mode 100755 modules/gerrit/files/scripts/update_bug.py delete mode 100755 modules/gerrit/files/scripts/update_cla_group.py delete mode 100755 modules/github/files/scripts/close_pull_requests.py create mode 100644 modules/jeepyb/manifests/init.pp delete mode 100644 modules/openstack_project/files/gerrit/scripts/trivial_rebase.py delete mode 100644 modules/pypimirror/files/process_cache.py delete mode 100644 modules/pypimirror/files/pull-repo.sh delete mode 100755 modules/pypimirror/files/run_mirror.py diff --git a/doc/source/gerrit.rst b/doc/source/gerrit.rst index c3104246a3..f0eca0bfcc 100644 --- a/doc/source/gerrit.rst +++ b/doc/source/gerrit.rst @@ -452,7 +452,7 @@ to use this build step. Auto Review Expiry ================== -Puppet automatically installs a daily cron job called ``expire_old_reviews.py`` +Puppet automatically installs a daily cron job called ``expire-old-reviews`` onto the gerrit servers. This script follows two rules: #. If the review hasn't been touched in 2 weeks, mark as abandoned. diff --git a/modules/gerrit/files/scripts/expire_old_reviews.py b/modules/gerrit/files/scripts/expire_old_reviews.py deleted file mode 100644 index f9e908a67f..0000000000 --- a/modules/gerrit/files/scripts/expire_old_reviews.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This script is designed to expire old code reviews that have not been touched -# using the following rules: -# 1. if open and no activity in 2 weeks, expire -# 2. if negative comment and no activity in 1 week, expire - -import paramiko -import json -import logging -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument('user', help='The gerrit admin user') -parser.add_argument('ssh_key', help='The gerrit admin SSH key file') -options = parser.parse_args() - -GERRIT_USER = options.user -GERRIT_SSH_KEY = options.ssh_key - -logging.basicConfig(format='%(asctime)-6s: %(name)s - %(levelname)s - %(message)s', filename='/var/log/gerrit/expire_reviews.log') -logger= logging.getLogger('expire_reviews') -logger.setLevel(logging.INFO) - -logger.info('Starting expire reviews') -logger.info('Connecting to Gerrit') - -ssh = paramiko.SSHClient() -ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) -ssh.connect('localhost', username=GERRIT_USER, key_filename=GERRIT_SSH_KEY, port=29418) - -def expire_patch_set(patch_id, patch_subject, has_negative): - if has_negative: - message= 'code review expired after 1 week of no activity after a negative review, it can be restored using the \`Restore Change\` button under the Patch Set on the web interface' - else: - message= 'code review expired after 2 weeks of no activity, it can be restored using the \`Restore Change\` button under the Patch Set on the web interface' - command='gerrit review --abandon --message="{0}" {1}'.format(message, patch_id) - logger.info('Expiring: %s - %s: %s', patch_id, patch_subject, message) - stdin, stdout, stderr = ssh.exec_command(command) - if stdout.channel.recv_exit_status() != 0: - logger.error(stderr.read()) - -# Query all open with no activity for 2 weeks -logger.info('Searching no activity for 2 weeks') -stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --format JSON status:open age:2w') - -for line in stdout: - row= json.loads(line) - if not row.has_key('rowCount'): - expire_patch_set(row['currentPatchSet']['revision'], row['subject'], False) - -# Query all reviewed with no activity for 1 week -logger.info('Searching no activity on negative review for 1 week') -stdin, stdout, stderr = ssh.exec_command('gerrit query --current-patch-set --all-approvals --format JSON status:reviewed age:1w') - -for line in stdout: - row= json.loads(line) - if not row.has_key('rowCount'): - # Search for negative approvals - for approval in row['currentPatchSet']['approvals']: - if approval['value'] == '-1': - expire_patch_set(row['currentPatchSet']['revision'], row['subject'], True) - break - -logger.info('End expire review') diff --git a/modules/gerrit/files/scripts/fetch_remotes.py b/modules/gerrit/files/scripts/fetch_remotes.py deleted file mode 100755 index 6f8ab8b4ed..0000000000 --- a/modules/gerrit/files/scripts/fetch_remotes.py +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Fetch remotes reads a project config file called projects.yaml -# It should look like: - -# - homepage: http://openstack.org -# team-id: 153703 -# has-wiki: False -# has-issues: False -# has-downloads: False -# --- -# - project: PROJECT_NAME -# options: -# - remote: https://gerrit.googlesource.com/gerrit - - -import logging -import os -import subprocess -import shlex -import yaml - -def run_command(cmd, status=False, env={}): - cmd_list = shlex.split(str(cmd)) - newenv = os.environ - newenv.update(env) - p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=newenv) - (out, nothing) = p.communicate() - if status: - return (p.returncode, out.strip()) - return out.strip() - - -def run_command_status(cmd, env={}): - return run_command(cmd, True, env) - - -logging.basicConfig(level=logging.ERROR) - -REPO_ROOT = os.environ.get('REPO_ROOT', - '/home/gerrit2/review_site/git') -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') - -(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))] - -for section in config: - project = section['project'] - - if 'remote' not in section: - continue - - project_git = "%s.git" % project - os.chdir(os.path.join(REPO_ROOT, project_git)) - - # Make sure that the specified remote exists - remote_url = section['remote'] - # We could check if it exists first, but we're ignoring output anyway - # So just try to make it, and it'll either make a new one or do nothing - run_command("git remote add -f upstream %s" % remote_url) - # Fetch new revs from it - run_command("git remote update upstream") diff --git a/modules/gerrit/files/scripts/get_group_uuid.py b/modules/gerrit/files/scripts/get_group_uuid.py deleted file mode 100644 index 335e944089..0000000000 --- a/modules/gerrit/files/scripts/get_group_uuid.py +++ /dev/null @@ -1,29 +0,0 @@ -import argparse -import paramiko -import json - -parser = argparse.ArgumentParser() -parser.add_argument("--host", dest="host", default="review.openstack.org", - help="gerrit host to connect to") -parser.add_argument("--port", dest="port", action='store', type=int, - default=29418, help="gerrit port to connect to") -parser.add_argument("groups", nargs=1) - -options = parser.parse_args() - - -client = paramiko.SSHClient() -client.load_system_host_keys() -client.set_missing_host_key_policy(paramiko.WarningPolicy()) -client.connect(options.host, port=options.port) - -group = options.groups[0] -query = "select group_uuid from account_groups where name = '%s'" % group -command = 'gerrit gsql --format JSON -c "%s"' % query -stdin, stdout, stderr = client.exec_command(command) - -for line in stdout: - row = json.loads(line) - if row['type'] == 'row': - print row['columns']['group_uuid'] - ret = stdout.channel.recv_exit_status() diff --git a/modules/gerrit/files/scripts/manage_projects.py b/modules/gerrit/files/scripts/manage_projects.py deleted file mode 100755 index 9f5c518085..0000000000 --- a/modules/gerrit/files/scripts/manage_projects.py +++ /dev/null @@ -1,327 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# Copyright (c) 2012 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# manage_projects.py reads a project config file called projects.yaml -# It should look like: - -# - homepage: http://openstack.org -# gerrit-host: review.openstack.org -# local-git-dir: /var/lib/git -# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key -# has-wiki: False -# has-issues: False -# has-downloads: False -# --- -# - project: PROJECT_NAME -# options: -# - has-wiki -# - has-issues -# - has-downloads -# - has-pull-requests -# homepage: Some homepage that isn't http://openstack.org -# description: This is a great project -# remote: https://gerrit.googlesource.com/gerrit -# upstream: git://github.com/bushy/beards.git -# acl_config: /path/to/gerrit/project.config - - -import ConfigParser -import logging -import os -import re -import shlex -import subprocess -import tempfile -import yaml - -import github -import gerritlib.gerrit - - -logging.basicConfig(level=logging.ERROR) -log = logging.getLogger("manage_projects") - - -def run_command(cmd, status=False, env={}): - cmd_list = shlex.split(str(cmd)) - newenv = os.environ - newenv.update(env) - log.debug("Executing command: %s" % " ".join(cmd_list)) - p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=newenv) - (out, nothing) = p.communicate() - log.debug("Return code: %s" % p.returncode) - log.debug("Command said: %s" % out.strip()) - if status: - return (p.returncode, out.strip()) - return out.strip() - - -def run_command_status(cmd, env={}): - return run_command(cmd, True, env) - - -def git_command(repo_dir, sub_cmd, env={}): - git_dir = os.path.join(repo_dir, '.git') - cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) - status, _ = run_command(cmd, True, env) - return status - - -def git_command_output(repo_dir, sub_cmd, env={}): - git_dir = os.path.join(repo_dir, '.git') - cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) - status, out = run_command(cmd, True, env) - return (status, out) - - -def fetch_config(project, remote_url, repo_path, env={}): - status = git_command(repo_path, "fetch %s +refs/meta/config:" - "refs/remotes/gerrit-meta/config" % remote_url, env) - if status != 0: - print "Failed to fetch refs/meta/config for project: %s" % project - return False - # Because the following fails if executed more than once you should only - # run fetch_config once in each repo. - status = git_command(repo_path, "checkout -b config " - "remotes/gerrit-meta/config") - if status != 0: - print "Failed to checkout config for project: %s" % project - return False - - return True - - -def copy_acl_config(project, repo_path, acl_config): - if not os.path.exists(acl_config): - return False - - acl_dest = os.path.join(repo_path, "project.config") - status, _ = run_command("cp %s %s" % - (acl_config, acl_dest), status=True) - if status == 0: - status = git_command(repo_path, "diff --quiet HEAD") - if status != 0: - return True - return False - - -def push_acl_config(project, remote_url, repo_path, env={}): - cmd = "commit -a -m'Update project config.' --author='Openstack Project " \ - "Creator '" - status = git_command(repo_path, cmd) - if status != 0: - print "Failed to commit config for project: %s" % project - return False - status, out = git_command_output(repo_path, - "push %s HEAD:refs/meta/config" % - remote_url, env) - if status != 0: - print "Failed to push config for project: %s" % project - print out - return False - return True - - -def _get_group_uuid(gerrit, group): - query = "select group_uuid from account_groups where name = '%s'" % group - data = gerrit.dbQuery(query) - if data: - for row in data: - if row["type"] == "row": - return row["columns"]["group_uuid"] - return None - - -def get_group_uuid(gerrit, group): - uuid = _get_group_uuid(gerrit, group) - if uuid: - return uuid - gerrit.createGroup(group) - uuid = _get_group_uuid(gerrit, group) - if uuid: - return uuid - return None - - -def create_groups_file(project, gerrit, repo_path): - acl_config = os.path.join(repo_path, "project.config") - group_file = os.path.join(repo_path, "groups") - uuids = {} - for line in open(acl_config, 'r'): - r = re.match(r'^\s+.*group\s+(.*)$', line) - if r: - group = r.group(1) - if group in uuids.keys(): - continue - uuid = get_group_uuid(gerrit, group) - if uuid: - uuids[group] = uuid - else: - return False - if uuids: - with open(group_file, 'w') as fp: - for group, uuid in uuids.items(): - fp.write("%s\t%s\n" % (uuid, group)) - status = git_command(repo_path, "add groups") - if status != 0: - print "Failed to add groups file for project: %s" % project - return False - return True - - -def make_ssh_wrapper(gerrit_user, gerrit_key): - (fd, name) = tempfile.mkstemp(text=True) - os.write(fd, '#!/bin/bash\n') - os.write(fd, - 'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' % - (gerrit_key, gerrit_user)) - os.close(fd) - os.chmod(name, 755) - return dict(GIT_SSH=name) - - -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') -configs = [config for config in yaml.load_all(open(PROJECTS_YAML))] -defaults = configs[0][0] -default_has_issues = defaults.get('has-issues', False) -default_has_downloads = defaults.get('has-downloads', False) -default_has_wiki = defaults.get('has-wiki', False) - -LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git') -GERRIT_HOST = defaults.get('gerrit-host') -GERRIT_USER = defaults.get('gerrit-user') -GERRIT_KEY = defaults.get('gerrit-key') -GITHUB_SECURE_CONFIG = defaults.get('github-config', - '/etc/github/github-projects.secure.config') - -secure_config = ConfigParser.ConfigParser() -secure_config.read(GITHUB_SECURE_CONFIG) - -# Project creation doesn't work via oauth -ghub = github.Github(secure_config.get("github", "username"), - secure_config.get("github", "password")) -orgs = ghub.get_user().get_orgs() -orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) - -gerrit = gerritlib.gerrit.Gerrit('localhost', - GERRIT_USER, - 29418, - GERRIT_KEY) -project_list = gerrit.listProjects() -ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) -try: - - for section in configs[1]: - project = section['project'] - options = section.get('options', dict()) - description = section.get('description', None) - homepage = section.get('homepage', defaults.get('homepage', None)) - upstream = section.get('upstream', None) - - project_git = "%s.git" % project - project_dir = os.path.join(LOCAL_GIT_DIR, project_git) - - # Find the project's repo - project_split = project.split('/', 1) - if len(project_split) > 1: - repo_name = project_split[1] - else: - repo_name = project - has_issues = 'has-issues' in options or default_has_issues - has_downloads = 'has-downloads' in options or default_has_downloads - has_wiki = 'has-wiki' in options or default_has_wiki - try: - org = orgs_dict[project_split[0].lower()] - except KeyError: - # We do not have control of this github org ignore the project. - continue - try: - repo = org.get_repo(repo_name) - except github.GithubException: - repo = org.create_repo(repo_name, - homepage=homepage, - has_issues=has_issues, - has_downloads=has_downloads, - has_wiki=has_wiki) - if description: - repo.edit(repo_name, description=description) - if homepage: - repo.edit(repo_name, homepage=homepage) - - repo.edit(repo_name, has_issues=has_issues, - has_downloads=has_downloads, - has_wiki=has_wiki) - - if 'gerrit' not in [team.name for team in repo.get_teams()]: - teams = org.get_teams() - teams_dict = dict(zip([t.name.lower() for t in teams], teams)) - teams_dict['gerrit'].add_to_repos(repo) - - remote_url = "ssh://localhost:29418/%s" % project - if project not in project_list: - tmpdir = tempfile.mkdtemp() - try: - repo_path = os.path.join(tmpdir, 'repo') - if upstream: - run_command("git clone %(upstream)s %(repo_path)s" % - dict(upstream=upstream, repo_path=repo_path)) - else: - run_command("git init %s" % repo_path) - with open(os.path.join(repo_path, - ".gitreview"), 'w') as gitreview: - gitreview.write(""" -[gerrit] -host=%s -port=29418 -project=%s -""" % (GERRIT_HOST, project_git)) - git_command(repo_path, "add .gitreview") - cmd = "commit -a -m'Added .gitreview' --author=" \ - "'Openstack Project Creator " \ - "'" - git_command(repo_path, cmd) - gerrit.createProject(project) - - if not os.path.exists(project_dir): - run_command("git --bare init %s" % project_dir) - run_command("chown -R gerrit2:gerrit2 %s" % project_dir) - - git_command(repo_path, - "push --all %s" % remote_url, - env=ssh_env) - git_command(repo_path, - "push --tags %s" % remote_url, env=ssh_env) - finally: - run_command("rm -fr %s" % tmpdir) - - if 'acl_config' in section: - tmpdir = tempfile.mkdtemp() - try: - repo_path = os.path.join(tmpdir, 'repo') - ret, _ = run_command_status("git init %s" % repo_path) - if ret != 0: - continue - if (fetch_config(project, remote_url, repo_path, ssh_env) and - copy_acl_config(project, repo_path, - section['acl_config']) and - create_groups_file(project, gerrit, repo_path)): - push_acl_config(project, remote_url, repo_path, ssh_env) - finally: - run_command("rm -fr %s" % tmpdir) -finally: - os.unlink(ssh_env['GIT_SSH']) diff --git a/modules/gerrit/files/scripts/notify_impact.py b/modules/gerrit/files/scripts/notify_impact.py deleted file mode 100755 index 5c8ba8c76e..0000000000 --- a/modules/gerrit/files/scripts/notify_impact.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is designed to be called by a gerrit hook. It searched new -# patchsets for strings like "bug FOO" and updates corresponding Launchpad -# bugs status. - -import argparse -import re -import subprocess -import smtplib - -from email.mime.text import MIMEText - -BASE_DIR = '/home/gerrit2/review_site' -EMAIL_TEMPLATE = """ -Hi, I'd like you to take a look at this patch for potential -%s. -%s - -Log: -%s -""" - -def process_impact(git_log, args): - """Notify mail list of impact""" - email_content = EMAIL_TEMPLATE % (args.impact, args.change_url, git_log) - msg = MIMEText(email_content) - msg['Subject'] = '[%s] %s review request change %s' % \ - (args.project, args.impact, args.change) - msg['From'] = 'gerrit2@review.openstack.org' - msg['To'] = args.dest_address - - s = smtplib.SMTP('localhost') - s.sendmail('gerrit2@review.openstack.org', - args.dest_address, msg.as_string()) - s.quit() - -def impacted(git_log, impact_string): - """Determine if a changes log indicates there is an impact""" - return re.search(impact_string, git_log, re.IGNORECASE) - -def extract_git_log(args): - """Extract git log of all merged commits""" - cmd = ['git', - '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git', - 'log', '--no-merges', args.commit + '^1..' + args.commit] - return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('hook') - #common - parser.add_argument('--change', default=None) - parser.add_argument('--change-url', default=None) - parser.add_argument('--project', default=None) - parser.add_argument('--branch', default=None) - parser.add_argument('--commit', default=None) - #change-merged - parser.add_argument('--submitter', default=None) - #patchset-created - parser.add_argument('--uploader', default=None) - parser.add_argument('--patchset', default=None) - # Not passed by gerrit: - parser.add_argument('--impact', default=None) - parser.add_argument('--dest-address', default=None) - - args = parser.parse_args() - - # Get git log - git_log = extract_git_log(args) - - # Process impacts found in git log - if impacted(git_log, args.impact): - process_impact(git_log, args) - - -if __name__ == '__main__': - main() diff --git a/modules/gerrit/files/scripts/update_blueprint.py b/modules/gerrit/files/scripts/update_blueprint.py deleted file mode 100755 index be70ec2deb..0000000000 --- a/modules/gerrit/files/scripts/update_blueprint.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is designed to be called by a gerrit hook. It searched new -# patchsets for strings like "blueprint FOO" or "bp FOO" and updates -# corresponding Launchpad blueprints with links back to the change. - -from launchpadlib.launchpad import Launchpad -from launchpadlib.uris import LPNET_SERVICE_ROOT -import os -import argparse -import re -import subprocess - -import StringIO -import ConfigParser -import MySQLdb - -BASE_DIR = '/home/gerrit2/review_site' -GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR', - '~/.launchpadlib/cache')) -GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS', - '~/.launchpadlib/creds')) -GERRIT_CONFIG = os.environ.get('GERRIT_CONFIG', - '/home/gerrit2/review_site/etc/gerrit.config') -GERRIT_SECURE_CONFIG = os.environ.get('GERRIT_SECURE_CONFIG', - '/home/gerrit2/review_site/etc/secure.config') -SPEC_RE = re.compile(r'(blueprint|bp)\s*[#:]?\s*(\S+)', re.I) -BODY_RE = re.compile(r'^\s+.*$') - -def get_broken_config(filename): - """ gerrit config ini files are broken and have leading tabs """ - text = "" - with open(filename,"r") as conf: - for line in conf.readlines(): - text = "%s%s" % (text, line.lstrip()) - - fp = StringIO.StringIO(text) - c=ConfigParser.ConfigParser() - c.readfp(fp) - return c - -GERRIT_CONFIG = get_broken_config(GERRIT_CONFIG) -SECURE_CONFIG = get_broken_config(GERRIT_SECURE_CONFIG) -DB_USER = GERRIT_CONFIG.get("database", "username") -DB_PASS = SECURE_CONFIG.get("database","password") -DB_DB = GERRIT_CONFIG.get("database","database") - -def update_spec(launchpad, project, name, subject, link, topic=None): - group, project = project.split('/') - spec = launchpad.projects[project].getSpecification(name=name) - if not spec: return - - if spec.whiteboard: - wb = spec.whiteboard.strip() - else: - wb = '' - changed = False - if topic: - topiclink = '%s/#q,topic:%s,n,z' % (link[:link.find('/',8)], - topic) - if topiclink not in wb: - wb += "\n\n\nGerrit topic: %(link)s" % dict(link=topiclink) - changed = True - - if link not in wb: - wb += "\n\n\nAddressed by: %(link)s\n %(subject)s\n" % dict(subject=subject, - link=link) - changed = True - - if changed: - spec.whiteboard = wb - spec.lp_save() - -def find_specs(launchpad, dbconn, args): - git_log = subprocess.Popen(['git', - '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git', - 'log', '--no-merges', - args.commit + '^1..' + args.commit], - stdout=subprocess.PIPE).communicate()[0] - - cur = dbconn.cursor() - cur.execute("select subject, topic from changes where change_key=%s", args.change) - subject, topic = cur.fetchone() - specs = set([m.group(2) for m in SPEC_RE.finditer(git_log)]) - - if topic: - topicspec = topic.split('/')[-1] - specs |= set([topicspec]) - - for spec in specs: - update_spec(launchpad, args.project, spec, subject, - args.change_url, topic) - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('hook') - #common - parser.add_argument('--change', default=None) - parser.add_argument('--change-url', default=None) - parser.add_argument('--project', default=None) - parser.add_argument('--branch', default=None) - parser.add_argument('--commit', default=None) - #change-merged - parser.add_argument('--submitter', default=None) - # patchset-created - parser.add_argument('--uploader', default=None) - parser.add_argument('--patchset', default=None) - - args = parser.parse_args() - - launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT, - GERRIT_CACHE_DIR, - credentials_file = GERRIT_CREDENTIALS, - version='devel') - - conn = MySQLdb.connect(user = DB_USER, passwd = DB_PASS, db = DB_DB) - - find_specs(launchpad, conn, args) - -if __name__ == '__main__': - main() diff --git a/modules/gerrit/files/scripts/update_bug.py b/modules/gerrit/files/scripts/update_bug.py deleted file mode 100755 index 9479a81225..0000000000 --- a/modules/gerrit/files/scripts/update_bug.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is designed to be called by a gerrit hook. It searched new -# patchsets for strings like "bug FOO" and updates corresponding Launchpad -# bugs status. - -from launchpadlib.launchpad import Launchpad -from launchpadlib.uris import LPNET_SERVICE_ROOT -import os -import argparse -import re -import subprocess - - -BASE_DIR = '/home/gerrit2/review_site' -GERRIT_CACHE_DIR = os.path.expanduser(os.environ.get('GERRIT_CACHE_DIR', - '~/.launchpadlib/cache')) -GERRIT_CREDENTIALS = os.path.expanduser(os.environ.get('GERRIT_CREDENTIALS', - '~/.launchpadlib/creds')) - - -def add_change_proposed_message(bugtask, change_url, project, branch): - subject = 'Fix proposed to %s (%s)' % (short_project(project), branch) - body = 'Fix proposed to branch: %s\nReview: %s' % (branch, change_url) - bugtask.bug.newMessage(subject=subject, content=body) - - -def add_change_merged_message(bugtask, change_url, project, commit, - submitter, branch, git_log): - subject = 'Fix merged to %s (%s)' % (short_project(project), branch) - git_url = 'http://github.com/%s/commit/%s' % (project, commit) - body = '''Reviewed: %s -Committed: %s -Submitter: %s -Branch: %s\n''' % (change_url, git_url, submitter, branch) - body = body + '\n' + git_log - bugtask.bug.newMessage(subject=subject, content=body) - - -def set_in_progress(bugtask, launchpad, uploader, change_url): - """Set bug In progress with assignee being the uploader""" - - # Retrieve uploader from Launchpad. Use email as search key if - # provided, and only set if there is a clear match. - try: - searchkey = uploader[uploader.rindex("(") + 1:-1] - except ValueError: - searchkey = uploader - persons = launchpad.people.findPerson(text=searchkey) - if len(persons) == 1: - bugtask.assignee = persons[0] - - bugtask.status = "In Progress" - bugtask.lp_save() - - -def set_fix_committed(bugtask): - """Set bug fix committed""" - - bugtask.status = "Fix Committed" - bugtask.lp_save() - - -def set_fix_released(bugtask): - """Set bug fix released""" - - bugtask.status = "Fix Released" - bugtask.lp_save() - - -def release_fixcommitted(bugtask): - """Set bug FixReleased if it was FixCommitted""" - - if bugtask.status == u'Fix Committed': - set_fix_released(bugtask) - - -def tag_in_branchname(bugtask, branch): - """Tag bug with in-branch-name tag (if name is appropriate)""" - - lp_bug = bugtask.bug - branch_name = branch.replace('/', '-') - if branch_name.replace('-', '').isalnum(): - lp_bug.tags = lp_bug.tags + ["in-%s" % branch_name] - lp_bug.tags.append("in-%s" % branch_name) - lp_bug.lp_save() - - -def short_project(full_project_name): - """Return the project part of the git repository name""" - return full_project_name.split('/')[-1] - - -def git2lp(full_project_name): - """Convert Git repo name to Launchpad project""" - project_map = { - 'openstack/openstack-ci-puppet': 'openstack-ci', - 'openstack-ci/devstack-gate': 'openstack-ci', - 'openstack-ci/gerrit': 'openstack-ci', - 'openstack-ci/lodgeit': 'openstack-ci', - 'openstack-ci/meetbot': 'openstack-ci', - } - return project_map.get(full_project_name, short_project(full_project_name)) - - -def is_direct_release(full_project_name): - """Test against a list of projects who directly release changes.""" - return full_project_name in [ - 'openstack-ci/devstack-gate', - 'openstack-ci/lodgeit', - 'openstack-ci/meetbot', - 'openstack-dev/devstack', - 'openstack/openstack-ci', - 'openstack/openstack-ci-puppet', - 'openstack/openstack-manuals', - 'openstack/tempest', - ] - - -def process_bugtask(launchpad, bugtask, git_log, args): - """Apply changes to bugtask, based on hook / branch...""" - - if args.hook == "change-merged": - if args.branch == 'master': - if is_direct_release(args.project): - set_fix_released(bugtask) - else: - if bugtask.status != u'Fix Released': - set_fix_committed(bugtask) - elif args.branch == 'milestone-proposed': - release_fixcommitted(bugtask) - elif args.branch.startswith('stable/'): - series = args.branch[7:] - # Look for a related task matching the series - for reltask in bugtask.related_tasks: - if (reltask.bug_target_name.endswith("/" + series) and - reltask.status != u'Fix Released'): - # Use fixcommitted if there is any - set_fix_committed(reltask) - break - else: - # Use tagging if there isn't any - tag_in_branchname(bugtask, args.branch) - - add_change_merged_message(bugtask, args.change_url, args.project, - args.commit, args.submitter, args.branch, - git_log) - - if args.hook == "patchset-created": - if args.branch == 'master': - if bugtask.status not in [u'Fix Committed', u'Fix Released']: - set_in_progress(bugtask, launchpad, args.uploader, - args.change_url) - elif args.branch.startswith('stable/'): - series = args.branch[7:] - for reltask in bugtask.related_tasks: - if (reltask.bug_target_name.endswith("/" + series) and - reltask.status not in [u'Fix Committed', u'Fix Released']): - set_in_progress(reltask, launchpad, - args.uploader, args.change_url) - break - - if args.patchset == '1': - add_change_proposed_message(bugtask, args.change_url, - args.project, args.branch) - - -def find_bugs(launchpad, git_log, args): - """Find bugs referenced in the git log and return related bugtasks""" - - bug_regexp = r'([Bb]ug|[Ll][Pp])[\s#:]*(\d+)' - tokens = re.split(bug_regexp, git_log) - - # Extract unique bug tasks - bugtasks = {} - for token in tokens: - if re.match('^\d+$', token) and (token not in bugtasks): - try: - lp_bug = launchpad.bugs[token] - for lp_task in lp_bug.bug_tasks: - if lp_task.bug_target_name == git2lp(args.project): - bugtasks[token] = lp_task - break - except KeyError: - # Unknown bug - pass - - return bugtasks.values() - - -def extract_git_log(args): - """Extract git log of all merged commits""" - cmd = ['git', - '--git-dir=' + BASE_DIR + '/git/' + args.project + '.git', - 'log', '--no-merges', args.commit + '^1..' + args.commit] - return subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('hook') - #common - parser.add_argument('--change', default=None) - parser.add_argument('--change-url', default=None) - parser.add_argument('--project', default=None) - parser.add_argument('--branch', default=None) - parser.add_argument('--commit', default=None) - #change-merged - parser.add_argument('--submitter', default=None) - #patchset-created - parser.add_argument('--uploader', default=None) - parser.add_argument('--patchset', default=None) - - args = parser.parse_args() - - # Connect to Launchpad - launchpad = Launchpad.login_with('Gerrit User Sync', LPNET_SERVICE_ROOT, - GERRIT_CACHE_DIR, - credentials_file=GERRIT_CREDENTIALS, - version='devel') - - # Get git log - git_log = extract_git_log(args) - - # Process bugtasks found in git log - for bugtask in find_bugs(launchpad, git_log, args): - process_bugtask(launchpad, bugtask, git_log, args) - - -if __name__ == '__main__': - main() diff --git a/modules/gerrit/files/scripts/update_cla_group.py b/modules/gerrit/files/scripts/update_cla_group.py deleted file mode 100755 index 615b25688c..0000000000 --- a/modules/gerrit/files/scripts/update_cla_group.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Add launchpad ids listed in the wiki CLA page to the CLA group in LP. - -import os -import urllib -import re - -from launchpadlib.launchpad import Launchpad -from launchpadlib.uris import LPNET_SERVICE_ROOT - -DEBUG = False - -LP_CACHE_DIR = '~/.launchpadlib/cache' -LP_CREDENTIALS = '~/.launchpadlib/creds' -CONTRIBUTOR_RE = re.compile(r'.*?\|\|\s*(?P.*?)\s*\|\|\s*(?P.*?)\s*\|\|\s*(?P.*?)\s*\|\|.*?') -LINK_RE = re.compile(r'\[\[.*\|\s*(?P.*)\s*\]\]') - -for check_path in (os.path.dirname(LP_CACHE_DIR), - os.path.dirname(LP_CREDENTIALS)): - if not os.path.exists(check_path): - os.makedirs(check_path) - -wiki_members = [] -for line in urllib.urlopen('http://wiki.openstack.org/Contributors?action=raw'): - m = CONTRIBUTOR_RE.match(line) - if m and m.group('login') and m.group('trans'): - login = m.group('login') - if login=="<#c0c0c0>'''Launchpad ID'''": continue - l = LINK_RE.match(login) - if l: - login = l.group('name') - wiki_members.append(login) - -launchpad = Launchpad.login_with('CLA Team Sync', LPNET_SERVICE_ROOT, - LP_CACHE_DIR, - credentials_file = LP_CREDENTIALS, - version='devel') - -lp_members = [] - -team = launchpad.people['openstack-cla'] -for detail in team.members_details: - user = None - # detail.self_link == - # 'https://api.launchpad.net/1.0/~team/+member/${username}' - login = detail.self_link.split('/')[-1] - status = detail.status - lp_members.append(login) - -for wm in wiki_members: - if wm not in lp_members: - print "Need to add %s to LP" % (wm) - try: - person = launchpad.people[wm] - except: - print 'Unable to find %s on LP'%wm - continue - status = team.addMember(person=person, status="Approved") diff --git a/modules/gerrit/manifests/cron.pp b/modules/gerrit/manifests/cron.pp index 451c8521b4..d659110cfe 100644 --- a/modules/gerrit/manifests/cron.pp +++ b/modules/gerrit/manifests/cron.pp @@ -9,8 +9,8 @@ class gerrit::cron( user => 'gerrit2', hour => '6', minute => '3', - command => "python /usr/local/gerrit/scripts/expire_old_reviews.py ${script_user} ${script_key_file}", - require => File['/usr/local/gerrit/scripts'], + command => "python /usr/local/bin/expire-old-reviews ${script_user} ${script_key_file}", + require => Class['jeepyb'], } cron { 'gerrit_repack': diff --git a/modules/gerrit/manifests/init.pp b/modules/gerrit/manifests/init.pp index ded3577748..9305e93aca 100644 --- a/modules/gerrit/manifests/init.pp +++ b/modules/gerrit/manifests/init.pp @@ -111,6 +111,7 @@ class gerrit( $testmode = false ) { include apache + include jeepyb include pip $java_home = $::lsbdistcodename ? { @@ -490,13 +491,7 @@ class gerrit( } file { '/usr/local/gerrit/scripts': - ensure => directory, - owner => 'root', - group => 'root', - mode => '0755', - recurse => true, - require => File['/usr/local/gerrit'], - source => 'puppet:///modules/gerrit/scripts', + ensure => absent, } # Install Bouncy Castle's OpenPGP plugin and populate the contact store diff --git a/modules/gerrit/manifests/remotes.pp b/modules/gerrit/manifests/remotes.pp index ec3ac26b8c..9637f0b88b 100644 --- a/modules/gerrit/manifests/remotes.pp +++ b/modules/gerrit/manifests/remotes.pp @@ -5,8 +5,8 @@ class gerrit::remotes($ensure=present) { ensure => $ensure, user => 'gerrit2', minute => '*/30', - command => 'sleep $((RANDOM\%60+90)) && python /usr/local/gerrit/scripts/fetch_remotes.py', - require => File['/usr/local/gerrit/scripts'], + command => 'sleep $((RANDOM\%60+90)) && /usr/local/bin/fetch-remotes', + require => Class['jeepyb'], } file { '/home/gerrit2/remotes.config': diff --git a/modules/github/files/scripts/close_pull_requests.py b/modules/github/files/scripts/close_pull_requests.py deleted file mode 100755 index cecbd324cc..0000000000 --- a/modules/github/files/scripts/close_pull_requests.py +++ /dev/null @@ -1,94 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Github pull requests closer reads a project config file called projects.yaml -# It should look like: - -# - homepage: http://openstack.org -# team-id: 153703 -# has-wiki: False -# has-issues: False -# has-downloads: False -# --- -# - project: PROJECT_NAME -# options: -# - has-pull-requests - -# Github authentication information is read from github.secure.config, -# which should look like: - -# [github] -# username = GITHUB_USERNAME -# password = GITHUB_PASSWORD -# -# or -# -# [github] -# oauth_token = GITHUB_OAUTH_TOKEN - -import ConfigParser -import github -import os -import yaml -import logging - -logging.basicConfig(level=logging.ERROR) - -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') -GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG', - '/etc/github/github.secure.config') - -MESSAGE = """Thank you for contributing to OpenStack! - -%(project)s uses Gerrit for code review. - -Please visit http://wiki.openstack.org/GerritWorkflow and follow the instructions there to upload your change to Gerrit. -""" - -secure_config = ConfigParser.ConfigParser() -secure_config.read(GITHUB_SECURE_CONFIG) -(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))] - -if secure_config.has_option("github", "oauth_token"): - ghub = github.Github(secure_config.get("github", "oauth_token")) -else: - ghub = github.Github(secure_config.get("github", "username"), - secure_config.get("github", "password")) - -orgs = ghub.get_user().get_orgs() -orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) -for section in config: - project = section['project'] - - # Make sure we're supposed to close pull requests for this project: - if 'options' in section and 'has-pull-requests' in section['options']: - continue - - # Find the project's repo - project_split = project.split('/', 1) - if len(project_split) > 1: - repo = orgs_dict[project_split[0].lower()].get_repo(project_split[1]) - else: - repo = ghub.get_user().get_repo(project) - - # Close each pull request - pull_requests = repo.get_pulls("open") - for req in pull_requests: - vars = dict(project=project) - issue_data = {"url": repo.url + "/issues/" + str(req.number)} - issue = github.Issue.Issue(req._requester, issue_data, completed = True) - issue.create_comment(MESSAGE % vars) - req.edit(state = "closed") diff --git a/modules/github/manifests/init.pp b/modules/github/manifests/init.pp index dabceedab5..845a9c5aac 100644 --- a/modules/github/manifests/init.pp +++ b/modules/github/manifests/init.pp @@ -7,6 +7,7 @@ class github( $project_password = '', $projects = [] ) { + include jeepyb include pip package { 'PyGithub': @@ -80,20 +81,14 @@ class github( } file { '/usr/local/github/scripts': - ensure => directory, - group => 'root', - mode => '0755', - owner => 'root', - recurse => true, - require => File['/usr/local/github'], - source => 'puppet:///modules/github/scripts', + ensure => absent, } cron { 'githubclosepull': - command => 'sleep $((RANDOM\%60+90)) && python /usr/local/github/scripts/close_pull_requests.py', + command => 'sleep $((RANDOM\%60+90)) && /usr/local/bin/close-pull-requests', minute => '*/5', require => [ - File['/usr/local/github/scripts'], + Class['jeepyb'], Package['python-yaml'], Package['PyGithub'], ], diff --git a/modules/jeepyb/manifests/init.pp b/modules/jeepyb/manifests/init.pp new file mode 100644 index 0000000000..81801b86f8 --- /dev/null +++ b/modules/jeepyb/manifests/init.pp @@ -0,0 +1,54 @@ +# == Class: jeepyb +# +class jeepyb ( + $git_source_repo = 'https://github.com/openstack-ci/jeepyb.git', +) { + $packages = [ + 'python-mysqldb', + 'python-paramiko', + ] + + package { $packages: + ensure => present, + } + + if ! defined(Package['Pygithub']) { + package { 'PyGithub': + ensure => latest, + provider => pip, + require => Class['pip'], + } + } + + if ! defined(Package['gerritlib']) { + package { 'gerritlib': + ensure => latest, + provider => pip, + require => Class['pip'], + } + } + + # A lot of things need yaml, be conservative requiring this package to avoid + # conflicts with other modules. + if ! defined(Package['python-yaml']) { + package { 'python-yaml': + ensure => present, + } + } + + vcsrepo { '/opt/jeepyb': + ensure => latest, + provider => git, + revision => 'master', + source => $git_source_repo, + } + + exec { 'install_jeepyb' : + command => 'python setup.py install', + cwd => '/opt/jeepyb', + path => '/bin:/usr/bin', + refreshonly => true, + subscribe => Vcsrepo['/opt/jeepyb'], + } + +} diff --git a/modules/openstack_project/files/gerrit/change-merged b/modules/openstack_project/files/gerrit/change-merged index ef0cd3a296..a35424f302 100755 --- a/modules/openstack_project/files/gerrit/change-merged +++ b/modules/openstack_project/files/gerrit/change-merged @@ -1,4 +1,4 @@ #!/bin/sh # Use timeout to kill any process running longer than 10 minutes. -timeout -k 2m 10m python /usr/local/gerrit/scripts/update_bug.py change-merged "$@" +timeout -k 2m 10m /usr/local/bin/update-bug change-merged "$@" diff --git a/modules/openstack_project/files/gerrit/scripts/trivial_rebase.py b/modules/openstack_project/files/gerrit/scripts/trivial_rebase.py deleted file mode 100644 index 7575935232..0000000000 --- a/modules/openstack_project/files/gerrit/scripts/trivial_rebase.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python2.6 - -# Copyright (c) 2010, Code Aurora Forum. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# # Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# # Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# # Neither the name of Code Aurora Forum, Inc. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS -# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# This script is designed to detect when a patchset uploaded to Gerrit is -# 'identical' (determined via git-patch-id) and reapply reviews onto the new -# patchset from the previous patchset. - -# Get usage and help info by running: ./trivial_rebase.py --help -# Documentation is available here: https://www.codeaurora.org/xwiki/bin/QAEP/Gerrit - -import json -import subprocess -from sys import exit - -from optparse import OptionParser as _realOptionParser, AmbiguousOptionError, \ - BadOptionError -class OptionParser(_realOptionParser): - """Make OptionParser silently swallow unrecognized options.""" - def _process_args(self, largs, rargs, values): - while rargs: - try: - _realOptionParser._process_args(self, largs, rargs, values) - except (AmbiguousOptionError, BadOptionError), e: - largs.append(e.opt_str) - -class CheckCallError(OSError): - """CheckCall() returned non-0.""" - def __init__(self, command, cwd, retcode, stdout, stderr=None): - OSError.__init__(self, command, cwd, retcode, stdout, stderr) - self.command = command - self.cwd = cwd - self.retcode = retcode - self.stdout = stdout - self.stderr = stderr - -def CheckCall(command, cwd=None): - """Like subprocess.check_call() but returns stdout. - - Works on python 2.4 - """ - try: - process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE) - std_out, std_err = process.communicate() - except OSError, e: - raise CheckCallError(command, cwd, e.errno, None) - if process.returncode: - raise CheckCallError(command, cwd, process.returncode, std_out, std_err) - return std_out, std_err - -def Gssh(options, api_command): - """Makes a Gerrit API call via SSH and returns the stdout results.""" - ssh_cmd = ['ssh', - '-l', 'Gerrit Code Review', - '-p', options.port, - '-i', options.private_key_path, - options.server, - api_command] - try: - return CheckCall(ssh_cmd)[0] - except CheckCallError, e: - import sys - err_template = "call: %s\nreturn code: %s\nstdout: %s\nstderr: %s\n" - sys.stderr.write(err_template%(ssh_cmd, e.retcode, e.stdout, e.stderr)) - raise - -def GsqlQuery(sql_query, options): - """Runs a gerrit gsql query and returns the result""" - gsql_cmd = "gerrit gsql --format JSON -c %s"%sql_query - gsql_out = Gssh(options, gsql_cmd) - new_out = gsql_out.replace('}}\n', '}}\nsplit here\n') - return new_out.split('split here\n') - -def FindPrevRev(options): - """Finds the revision of the previous patch set on the change""" - sql_query = ("\"SELECT revision FROM patch_sets,changes WHERE " - "patch_sets.change_id = changes.change_id AND " - "patch_sets.patch_set_id = %s AND " - "changes.change_key = \'%s\'\"" % ((options.patchset - 1), - options.changeId)) - revisions = GsqlQuery(sql_query, options) - - json_dict = json.loads(revisions[0], strict=False) - return json_dict["columns"]["revision"] - -def GetApprovals(options): - """Get all the approvals on a specific patch set - - Returns a list of approval dicts""" - sql_query = ("\"SELECT value,account_id,category_id FROM patch_set_approvals " - "WHERE patch_set_id = %s AND change_id = (SELECT change_id FROM " - "changes WHERE change_key = \'%s\') AND value <> 0\"" - % ((options.patchset - 1), options.changeId)) - gsql_out = GsqlQuery(sql_query, options) - approvals = [] - for json_str in gsql_out: - dict = json.loads(json_str, strict=False) - if dict["type"] == "row": - approvals.append(dict["columns"]) - return approvals - -def GetPatchId(revision, consider_whitespace=False): - git_show_cmd = ['git', 'show', revision] - patch_id_cmd = ['git', 'patch-id'] - patch_id_process = subprocess.Popen(patch_id_cmd, stdout=subprocess.PIPE, - stdin=subprocess.PIPE) - git_show_process = subprocess.Popen(git_show_cmd, stdout=subprocess.PIPE) - if consider_whitespace: - # This matches on change lines in the patch (those starting with "+" - # or "-" but not followed by another of the same), then replaces any - # space or tab characters with "%" before calculating a patch-id. - replace_ws_cmd = ['sed', r'/^\(+[^+]\|-[^-]\)/y/ \t/%%/'] - replace_ws_process = subprocess.Popen(replace_ws_cmd, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE) - return patch_id_process.communicate( - replace_ws_process.communicate(git_show_process.communicate()[0])[0] - )[0] - else: - return patch_id_process.communicate(git_show_process.communicate()[0])[0] - -def SuExec(options, as_user, cmd): - suexec_cmd = "suexec --as %s -- %s"%(as_user, cmd) - Gssh(options, suexec_cmd) - -def DiffCommitMessages(commit1, commit2): - log_cmd1 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"', - commit1 + '^!'] - commit1_log = CheckCall(log_cmd1) - log_cmd2 = ['git', 'log', '--pretty=format:"%an %ae%n%s%n%b"', - commit2 + '^!'] - commit2_log = CheckCall(log_cmd2) - if commit1_log != commit2_log: - return True - return False - -def Main(): - usage = "usage: %prog [optional options]" - parser = OptionParser(usage=usage) - parser.add_option("--change", dest="changeId", help="Change identifier") - parser.add_option("--project", help="Project path in Gerrit") - parser.add_option("--commit", help="Git commit-ish for this patchset") - parser.add_option("--patchset", type="int", help="The patchset number") - parser.add_option("--role-user", dest="role_user", - help="E-mail/ID of user commenting on commit messages") - parser.add_option("--private-key-path", dest="private_key_path", - help="Full path to Gerrit SSH daemon's private host key") - parser.add_option("--server-port", dest="port", default='29418', - help="Port to connect to Gerrit's SSH daemon " - "[default: %default]") - parser.add_option("--server", dest="server", default="localhost", - help="Server name/address for Gerrit's SSH daemon " - "[default: %default]") - parser.add_option("--whitespace", action="store_true", - help="Treat whitespace as significant") - - (options, args) = parser.parse_args() - - if not options.changeId: - parser.print_help() - exit(0) - - if options.patchset == 1: - # Nothing to detect on first patchset - exit(0) - prev_revision = None - prev_revision = FindPrevRev(options) - if not prev_revision: - # Couldn't find a previous revision - exit(0) - prev_patch_id = GetPatchId(prev_revision) - cur_patch_id = GetPatchId(options.commit) - if cur_patch_id.split()[0] != prev_patch_id.split()[0]: - # patch-ids don't match - exit(0) - # Patch ids match. This is a trivial rebase. - # In addition to patch-id we should check if whitespace content changed. Some - # languages are more sensitive to whitespace than others, and some changes - # may either introduce or be intended to fix style problems specifically - # involving whitespace as well. - if options.whitespace: - prev_patch_ws = GetPatchId(prev_revision, consider_whitespace=True) - cur_patch_ws = GetPatchId(options.commit, consider_whitespace=True) - if cur_patch_ws.split()[0] != prev_patch_ws.split()[0]: - # Insert a comment into the change letting the approvers know only the - # whitespace changed - comment_msg = "\"New patchset patch-id matches previous patchset, " \ - "but whitespace content has changed.\"" - comment_cmd = ['gerrit', 'approve', '--project', options.project, - '--message', comment_msg, options.commit] - SuExec(options, options.role_user, ' '.join(comment_cmd)) - exit(0) - - # We should also check if the commit message changed. Most approvers would - # want to re-review changes when the commit message changes. - changed = DiffCommitMessages(prev_revision, options.commit) - if changed: - # Insert a comment into the change letting the approvers know only the - # commit message changed - comment_msg = "\"New patchset patch-id matches previous patchset, " \ - "but commit message has changed.\"" - comment_cmd = ['gerrit', 'approve', '--project', options.project, - '--message', comment_msg, options.commit] - SuExec(options, options.role_user, ' '.join(comment_cmd)) - exit(0) - - # Need to get all approvals on prior patch set, then suexec them onto - # this patchset. - approvals = GetApprovals(options) - gerrit_approve_msg = ("\'Automatically re-added by Gerrit trivial rebase " - "detection script.\'") - for approval in approvals: - # Note: Sites with different 'copy_min_score' values in the - # approval_categories DB table might want different behavior here. - # Additional categories should also be added if desired. - if approval["category_id"] == "CRVW": - approve_category = '--code-review' - elif approval["category_id"] == "VRIF": - # Don't re-add verifies - #approve_category = '--verified' - continue - elif approval["category_id"] == "SUBM": - # We don't care about previous submit attempts - continue - elif approval["category_id"] == "APRV": - # Similarly squash old approvals - continue - else: - print "Unsupported category: %s" % approval - exit(0) - - score = approval["value"] - gerrit_approve_cmd = ['gerrit', 'approve', '--project', options.project, - '--message', gerrit_approve_msg, approve_category, - score, options.commit] - SuExec(options, approval["account_id"], ' '.join(gerrit_approve_cmd)) - exit(0) - -if __name__ == "__main__": - Main() diff --git a/modules/openstack_project/manifests/gerrit.pp b/modules/openstack_project/manifests/gerrit.pp index 76e07a8c09..bed9997927 100644 --- a/modules/openstack_project/manifests/gerrit.pp +++ b/modules/openstack_project/manifests/gerrit.pp @@ -214,17 +214,6 @@ class openstack_project::gerrit ( require => Class['::gerrit'], } - file { '/usr/local/gerrit/scripts/trivial_rebase.py': - ensure => present, - owner => 'root', - group => 'root', - mode => '0444', - source => - 'puppet:///modules/openstack_project/gerrit/scripts/trivial_rebase.py', - replace => true, - require => Class['::gerrit'], - } - if ($projects_file != 'UNDEF') { if ($replicate_local) { file { $local_git_dir: @@ -264,7 +253,7 @@ class openstack_project::gerrit ( } exec { 'manage_projects': - command => '/usr/local/gerrit/scripts/manage_projects.py', + command => '/usr/local/bin/manage-projects', subscribe => [ File['/home/gerrit2/projects.yaml'], File['/home/gerrit2/acls'], @@ -273,6 +262,7 @@ class openstack_project::gerrit ( require => [ File['/home/gerrit2/projects.yaml'], File['/home/gerrit2/acls'], + Class['jeepyb'], ], } } diff --git a/modules/openstack_project/templates/gerrit_patchset-created.erb b/modules/openstack_project/templates/gerrit_patchset-created.erb index fabfceebf6..13f5fbbe38 100755 --- a/modules/openstack_project/templates/gerrit_patchset-created.erb +++ b/modules/openstack_project/templates/gerrit_patchset-created.erb @@ -1,11 +1,11 @@ #!/bin/sh # Use timeout to kill any process running longer than 10 minutes. -timeout -k 2m 10m python /usr/local/gerrit/scripts/update_blueprint.py patchset-created "$@" -timeout -k 2m 10m python /usr/local/gerrit/scripts/update_bug.py patchset-created "$@" -timeout -k 2m 10m python /usr/local/gerrit/scripts/notify_impact.py patchset-created "$@" --impact DocImpact --dest-address 'openstack-docs@lists.openstack.org' -timeout -k 2m 10m python /usr/local/gerrit/scripts/notify_impact.py patchset-created "$@" --impact SecurityImpact --dest-address 'openstack-ossg@lists.launchpad.net' -timeout -k 2m 10m python /usr/local/gerrit/scripts/trivial_rebase.py \ +timeout -k 2m 10m /usr/local/bin/update-blueprint patchset-created "$@" +timeout -k 2m 10m /usr/local/bin/update-bug patchset-created "$@" +timeout -k 2m 10m /usr/local/bin/notify-impact patchset-created "$@" --impact DocImpact --dest-address 'openstack-docs@lists.openstack.org' +timeout -k 2m 10m /usr/local/bin/notify-impact patchset-created "$@" --impact SecurityImpact --dest-address 'openstack-ossg@lists.launchpad.net' +timeout -k 2m 10m /usr/local/bin/trivial-rebase \ patchset-created \ --whitespace \ --private-key-path=<%= ssh_host_key %> \ diff --git a/modules/pypimirror/files/process_cache.py b/modules/pypimirror/files/process_cache.py deleted file mode 100644 index 2b4c054fc1..0000000000 --- a/modules/pypimirror/files/process_cache.py +++ /dev/null @@ -1,90 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import urllib -import datetime -import sys -import re -import md5 - -source_cache = sys.argv[1] -destination_mirror = sys.argv[2] - -PACKAGE_VERSION_RE = re.compile(r'(.*)-[0-9]') - -packages = {} -package_count = 0 - -for filename in os.listdir(source_cache): - if filename.endswith('content-type'): - continue - - realname = urllib.unquote(filename) - # The ? accounts for sourceforge downloads - tarball = os.path.basename(realname).split("?")[0] - name_match = PACKAGE_VERSION_RE.search(tarball) - - if name_match is None: - continue - package_name = name_match.group(1) - - version_list = packages.get(package_name,{}) - version_list[tarball] = filename - packages[package_name] = version_list - package_count = package_count + 1 - -full_html = open(os.path.join(destination_mirror, "full.html"), 'w') -simple_html = open(os.path.join(destination_mirror, "index.html"), 'w') - -header = "PyPI Mirror

PyPI Mirror

Last update: %s

\n\n" % datetime.datetime.utcnow().strftime("%c UTC") -full_html.write(header) -simple_html.write(header) - -for package_name, versions in packages.items(): - destination_dir = os.path.join(destination_mirror, package_name) - if not os.path.isdir(destination_dir): - os.makedirs(destination_dir) - safe_dir = urllib.quote(package_name) - simple_html.write("%s
\n" % (safe_dir, safe_dir)) - with open(os.path.join(destination_dir, "index.html"), 'w') as index: - index.write(""" - %s – PyPI Mirror -\n""" % package_name) - for tarball, filename in versions.items(): - source_path = os.path.join(source_cache, filename) - destination_path = os.path.join(destination_dir, tarball) - with open(destination_path, 'w') as dest: - src = open(source_path, 'r').read() - md5sum = md5.md5(src).hexdigest() - dest.write(src) - - safe_name = urllib.quote(tarball) - - full_html.write("%s
\n" % (safe_dir, - safe_name, - safe_name)) - index.write("%s\n" % (safe_name, - md5sum, - safe_name)) - index.write("\n") -footer = """

Generated by process_cache.py; %d -packages mirrored.

-\n""" % package_count -full_html.write(footer) -full_html.close() -simple_html.write(footer) -simple_html.close() diff --git a/modules/pypimirror/files/pull-repo.sh b/modules/pypimirror/files/pull-repo.sh deleted file mode 100644 index 4c1b12d672..0000000000 --- a/modules/pypimirror/files/pull-repo.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# This file is managed by puppet. -# https://github.com/openstack/openstack-ci-puppet - -export PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE:-/var/cache/pip} -export PIP_TEMP_DOWNLOAD=${PIP_TEMP_DOWNLOAD:-/var/lib/pip-download} - -project=$1 -pip_command='/usr/local/bin/pip install -M -U -I --exists-action=w --no-install' - -cd ${PIP_TEMP_DOWNLOAD} -short_project=`echo ${project} | cut -f2 -d/` -if [ ! -d ${short_project} ] ; then - git clone git://github.com/${project}.git ${short_project} >/dev/null 2>&1 -fi -cd ${short_project} -$pip_command pip -git fetch origin -for branch in `git branch -a | grep remotes.origin | grep -v origin.HEAD | awk '{print $1}' ` ; do - git reset --hard $branch - git clean -x -f -d -q - echo "*********************" - echo "Fetching pip requires for $project:$branch" - for requires_file in tools/pip-requires tools/test-requires ; do - if [ -f ${requires_file} ] ; then - $pip_command -r $requires_file - fi - done -done diff --git a/modules/pypimirror/files/run_mirror.py b/modules/pypimirror/files/run_mirror.py deleted file mode 100755 index b6b07e84c8..0000000000 --- a/modules/pypimirror/files/run_mirror.py +++ /dev/null @@ -1,81 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# run_mirrors reads a project config file called projects.yaml -# It should look like: - -# - project: PROJECT_NAME - -import logging -import os -import subprocess -import shlex -import yaml - -def run_command(cmd, status=False, env={}): - cmd_list = shlex.split(str(cmd)) - newenv = os.environ - newenv.update(env) - p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=newenv) - (out, nothing) = p.communicate() - if status: - return (p.returncode, out.strip()) - return out.strip() - - -def run_command_status(cmd, env={}): - return run_command(cmd, True, env) - - -logging.basicConfig(level=logging.ERROR) - -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/etc/openstackci/projects.yaml') -PIP_TEMP_DOWNLOAD = os.environ.get('PIP_TEMP_DOWNLOAD', - '/var/lib/pip-download') -GIT_SOURCE = os.environ.get('GIT_SOURCE', 'https://github.com') -pip_command = '/usr/local/bin/pip install -M -U -I --exists-action=w ' \ - '--no-install %s' - -run_command(pip_command % "pip") - -(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))] - -for section in config: - project = section['project'] - - os.chdir(PIP_TEMP_DOWNLOAD) - short_project = project.split('/')[1] - if not os.path.isdir(short_project): - run_command("git clone %s/%s.git %s" % (GIT_SOURCE, project, - short_project)) - os.chdir(short_project) - run_command("git fetch origin") - - for branch in run_command("git branch -a").split("\n"): - branch = branch.strip() - if (not branch.startswith("remotes/origin") - or "origin/HEAD" in branch): - continue - run_command("git reset --hard %s" % branch) - run_command("git clean -x -f -d -q") - print("*********************") - print("Fetching pip requires for %s:%s" % (project, branch)) - for requires_file in ("tools/pip-requires", "tools/test-requires"): - if os.path.exists(requires_file): - stanza = "-r %s" % requires_file - run_command(pip_command % stanza) - diff --git a/modules/pypimirror/manifests/init.pp b/modules/pypimirror/manifests/init.pp index e4acd2234c..f4856f067c 100644 --- a/modules/pypimirror/manifests/init.pp +++ b/modules/pypimirror/manifests/init.pp @@ -15,6 +15,7 @@ class pypimirror( include apache include pip include remove_nginx + include jeepyb package { 'python-yaml': ensure => present, @@ -68,30 +69,15 @@ class pypimirror( } file { '/usr/local/mirror_scripts/run_mirror.py': - ensure => present, - mode => '0755', - owner => 'root', - group => 'root', - source => 'puppet:///modules/pypimirror/run_mirror.py', - require => File['/usr/local/mirror_scripts'], + ensure => absent, } file { '/usr/local/mirror_scripts/pull-repo.sh': - ensure => present, - mode => '0755', - owner => 'root', - group => 'root', - source => 'puppet:///modules/pypimirror/pull-repo.sh', - require => File['/usr/local/mirror_scripts'], + ensure => absent, } file { '/usr/local/mirror_scripts/process_cache.py': - ensure => present, - mode => '0755', - owner => 'root', - group => 'root', - source => 'puppet:///modules/pypimirror/process_cache.py', - require => File['/usr/local/mirror_scripts'], + ensure => absent, } # Add cron job to update the mirror diff --git a/modules/pypimirror/templates/run-mirror.sh.erb b/modules/pypimirror/templates/run-mirror.sh.erb index 0c2b25bbcb..c6609d33f6 100644 --- a/modules/pypimirror/templates/run-mirror.sh.erb +++ b/modules/pypimirror/templates/run-mirror.sh.erb @@ -6,5 +6,5 @@ export PIP_DOWNLOAD_CACHE=<%= pip_cache %> export PIP_TEMP_DOWNLOAD=<%= pip_download %> export MIRROR_FILE_PATH=<%= mirror_file_path %> export LOG_FILENAME=<%= log_filename %> -python /usr/local/mirror_scripts/run_mirror.py <%= git_source %> >>$LOG_FILENAME -python /usr/local/mirror_scripts/process_cache.py ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH} +/usr/local/bin/run-mirror <%= git_source %> >>$LOG_FILENAME +/usr/local/bin/process-cache ${PIP_DOWNLOAD_CACHE} ${MIRROR_FILE_PATH}