333f4cb335
In the manage projects script git will sometimes report that there is a difference between old and new acl files when there isn't one. This causes the script to fail on those files when trying to commit them. Use git diff instead of git diff-index to resolve this problem. Change-Id: I9f2d7b3066a9bfe6e64282e9e7c009d4008e133e Reviewed-on: https://review.openstack.org/26172 Reviewed-by: Jeremy Stanley <fungi@yuggoth.org> Reviewed-by: James E. Blair <corvus@inaugust.com> Reviewed-by: Monty Taylor <mordred@inaugust.com> Approved: James E. Blair <corvus@inaugust.com> Tested-by: Jenkins
393 lines
14 KiB
Python
393 lines
14 KiB
Python
#! /usr/bin/env python
|
|
# Copyright (C) 2011 OpenStack, LLC.
|
|
# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
# manage_projects.py reads a project config file called projects.yaml
|
|
# It should look like:
|
|
|
|
# - homepage: http://openstack.org
|
|
# gerrit-host: review.openstack.org
|
|
# local-git-dir: /var/lib/git
|
|
# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
|
|
# has-wiki: False
|
|
# has-issues: False
|
|
# has-downloads: False
|
|
# acl-dir: /home/gerrit2/acls
|
|
# acl-base: /home/gerrit2/acls/project.config
|
|
# ---
|
|
# - project: PROJECT_NAME
|
|
# options:
|
|
# - has-wiki
|
|
# - has-issues
|
|
# - has-downloads
|
|
# - has-pull-requests
|
|
# homepage: Some homepage that isn't http://openstack.org
|
|
# description: This is a great project
|
|
# remote: https://gerrit.googlesource.com/gerrit
|
|
# upstream: git://github.com/bushy/beards.git
|
|
# acl-config: /path/to/gerrit/project.config
|
|
# acl-append:
|
|
# - /path/to/gerrit/project.config
|
|
# acl-parameters:
|
|
# project: OTHER_PROJECT_NAME
|
|
|
|
|
|
import ConfigParser
|
|
import logging
|
|
import os
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import tempfile
|
|
import yaml
|
|
|
|
import github
|
|
import gerritlib.gerrit
|
|
|
|
import jeepyb.gerritdb
|
|
|
|
logging.basicConfig(level=logging.ERROR)
|
|
log = logging.getLogger("manage_projects")
|
|
|
|
|
|
def run_command(cmd, status=False, env={}):
|
|
cmd_list = shlex.split(str(cmd))
|
|
newenv = os.environ
|
|
newenv.update(env)
|
|
log.debug("Executing command: %s" % " ".join(cmd_list))
|
|
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, env=newenv)
|
|
(out, nothing) = p.communicate()
|
|
log.debug("Return code: %s" % p.returncode)
|
|
log.debug("Command said: %s" % out.strip())
|
|
if status:
|
|
return (p.returncode, out.strip())
|
|
return out.strip()
|
|
|
|
|
|
def run_command_status(cmd, env={}):
|
|
return run_command(cmd, True, env)
|
|
|
|
|
|
def git_command(repo_dir, sub_cmd, env={}):
|
|
git_dir = os.path.join(repo_dir, '.git')
|
|
cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
|
|
status, _ = run_command(cmd, True, env)
|
|
return status
|
|
|
|
|
|
def git_command_output(repo_dir, sub_cmd, env={}):
|
|
git_dir = os.path.join(repo_dir, '.git')
|
|
cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
|
|
status, out = run_command(cmd, True, env)
|
|
return (status, out)
|
|
|
|
|
|
def write_acl_config(project, acl_dir, acl_base, acl_append, parameters):
|
|
project_parts = os.path.split(project)
|
|
if len(project_parts) > 1:
|
|
repo_base = os.path.join(acl_dir, *project_parts[:-1])
|
|
if not os.path.exists(repo_base):
|
|
os.makedirs(repo_base)
|
|
if not os.path.isdir(repo_base):
|
|
return 1
|
|
project = project_parts[-1]
|
|
config_file = os.path.join(repo_base, "%s.config" % project)
|
|
else:
|
|
config_file = os.path.join(acl_dir, "%s.config" % project)
|
|
if 'project' not in parameters:
|
|
parameters['project'] = project
|
|
with open(config_file, 'w') as config:
|
|
if acl_base and os.path.exists(acl_base):
|
|
config.write(open(acl_base, 'r').read())
|
|
for acl_snippet in acl_append:
|
|
if not os.path.exists(acl_snippet):
|
|
acl_snippet = os.path.join(acl_dir, acl_snippet)
|
|
if not os.path.exists(acl_snippet):
|
|
continue
|
|
with open(acl_snippet, 'r') as append_content:
|
|
config.write(append_content.read() % parameters)
|
|
|
|
|
|
def fetch_config(project, remote_url, repo_path, env={}):
|
|
status = git_command(repo_path, "fetch %s +refs/meta/config:"
|
|
"refs/remotes/gerrit-meta/config" % remote_url, env)
|
|
if status != 0:
|
|
print "Failed to fetch refs/meta/config for project: %s" % project
|
|
return False
|
|
# Because the following fails if executed more than once you should only
|
|
# run fetch_config once in each repo.
|
|
status = git_command(repo_path, "checkout -b config "
|
|
"remotes/gerrit-meta/config")
|
|
if status != 0:
|
|
print "Failed to checkout config for project: %s" % project
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def copy_acl_config(project, repo_path, acl_config):
|
|
if not os.path.exists(acl_config):
|
|
return False
|
|
|
|
acl_dest = os.path.join(repo_path, "project.config")
|
|
status, _ = run_command("cp %s %s" %
|
|
(acl_config, acl_dest), status=True)
|
|
if status == 0:
|
|
status = git_command(repo_path, "diff --quiet")
|
|
if status != 0:
|
|
return True
|
|
return False
|
|
|
|
|
|
def push_acl_config(project, remote_url, repo_path, env={}):
|
|
cmd = "commit -a -m'Update project config.' --author='Openstack Project " \
|
|
"Creator <openstack-infra@lists.openstack.org>'"
|
|
status = git_command(repo_path, cmd)
|
|
if status != 0:
|
|
print "Failed to commit config for project: %s" % project
|
|
return False
|
|
status, out = git_command_output(repo_path,
|
|
"push %s HEAD:refs/meta/config" %
|
|
remote_url, env)
|
|
if status != 0:
|
|
print "Failed to push config for project: %s" % project
|
|
print out
|
|
return False
|
|
return True
|
|
|
|
|
|
def _get_group_uuid(gerrit, group):
|
|
cursor = jeepyb.gerritdb.connect().cursor()
|
|
query = "SELECT group_uuid FROM account_groups WHERE name = %s"
|
|
cursor.execute(query, group)
|
|
data = cursor.fetchone()
|
|
if data:
|
|
return data[0]
|
|
return None
|
|
|
|
|
|
def get_group_uuid(gerrit, group):
|
|
uuid = _get_group_uuid(gerrit, group)
|
|
if uuid:
|
|
return uuid
|
|
gerrit.createGroup(group, owner="Administrators")
|
|
uuid = _get_group_uuid(gerrit, group)
|
|
if uuid:
|
|
return uuid
|
|
return None
|
|
|
|
|
|
def create_groups_file(project, gerrit, repo_path):
|
|
acl_config = os.path.join(repo_path, "project.config")
|
|
group_file = os.path.join(repo_path, "groups")
|
|
uuids = {}
|
|
for line in open(acl_config, 'r'):
|
|
r = re.match(r'^\s+.*group\s+(.*)$', line)
|
|
if r:
|
|
group = r.group(1)
|
|
if group in uuids.keys():
|
|
continue
|
|
uuid = get_group_uuid(gerrit, group)
|
|
if uuid:
|
|
uuids[group] = uuid
|
|
else:
|
|
return False
|
|
if uuids:
|
|
with open(group_file, 'w') as fp:
|
|
for group, uuid in uuids.items():
|
|
fp.write("%s\t%s\n" % (uuid, group))
|
|
status = git_command(repo_path, "add groups")
|
|
if status != 0:
|
|
print "Failed to add groups file for project: %s" % project
|
|
return False
|
|
return True
|
|
|
|
|
|
def make_ssh_wrapper(gerrit_user, gerrit_key):
|
|
(fd, name) = tempfile.mkstemp(text=True)
|
|
os.write(fd, '#!/bin/bash\n')
|
|
os.write(fd,
|
|
'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
|
|
(gerrit_key, gerrit_user))
|
|
os.close(fd)
|
|
os.chmod(name, 0755)
|
|
return dict(GIT_SSH=name)
|
|
|
|
|
|
def main():
|
|
|
|
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
|
|
'/home/gerrit2/projects.yaml')
|
|
configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
|
|
defaults = configs[0][0]
|
|
default_has_issues = defaults.get('has-issues', False)
|
|
default_has_downloads = defaults.get('has-downloads', False)
|
|
default_has_wiki = defaults.get('has-wiki', False)
|
|
|
|
LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
|
|
ACL_DIR = defaults.get('acl-dir')
|
|
GERRIT_HOST = defaults.get('gerrit-host')
|
|
GERRIT_USER = defaults.get('gerrit-user')
|
|
GERRIT_KEY = defaults.get('gerrit-key')
|
|
|
|
GITHUB_SECURE_CONFIG = defaults.get(
|
|
'github-config',
|
|
'/etc/github/github-projects.secure.config')
|
|
|
|
secure_config = ConfigParser.ConfigParser()
|
|
secure_config.read(GITHUB_SECURE_CONFIG)
|
|
|
|
# Project creation doesn't work via oauth
|
|
ghub = github.Github(secure_config.get("github", "username"),
|
|
secure_config.get("github", "password"))
|
|
orgs = ghub.get_user().get_orgs()
|
|
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
|
|
|
|
gerrit = gerritlib.gerrit.Gerrit('localhost',
|
|
GERRIT_USER,
|
|
29418,
|
|
GERRIT_KEY)
|
|
project_list = gerrit.listProjects()
|
|
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
|
|
try:
|
|
|
|
for section in configs[1]:
|
|
project = section['project']
|
|
options = section.get('options', dict())
|
|
description = section.get('description', None)
|
|
homepage = section.get('homepage', defaults.get('homepage', None))
|
|
upstream = section.get('upstream', None)
|
|
|
|
project_git = "%s.git" % project
|
|
project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
|
|
|
|
# Find the project's repo
|
|
project_split = project.split('/', 1)
|
|
if len(project_split) > 1:
|
|
repo_name = project_split[1]
|
|
else:
|
|
repo_name = project
|
|
has_issues = 'has-issues' in options or default_has_issues
|
|
has_downloads = 'has-downloads' in options or default_has_downloads
|
|
has_wiki = 'has-wiki' in options or default_has_wiki
|
|
try:
|
|
org = orgs_dict[project_split[0].lower()]
|
|
except KeyError:
|
|
# We do not have control of this github org ignore the project.
|
|
continue
|
|
try:
|
|
repo = org.get_repo(repo_name)
|
|
except github.GithubException:
|
|
repo = org.create_repo(repo_name,
|
|
homepage=homepage,
|
|
has_issues=has_issues,
|
|
has_downloads=has_downloads,
|
|
has_wiki=has_wiki)
|
|
if description:
|
|
repo.edit(repo_name, description=description)
|
|
if homepage:
|
|
repo.edit(repo_name, homepage=homepage)
|
|
|
|
repo.edit(repo_name, has_issues=has_issues,
|
|
has_downloads=has_downloads,
|
|
has_wiki=has_wiki)
|
|
|
|
if 'gerrit' not in [team.name for team in repo.get_teams()]:
|
|
teams = org.get_teams()
|
|
teams_dict = dict(zip([t.name.lower() for t in teams], teams))
|
|
teams_dict['gerrit'].add_to_repos(repo)
|
|
|
|
remote_url = "ssh://localhost:29418/%s" % project
|
|
if project not in project_list:
|
|
tmpdir = tempfile.mkdtemp()
|
|
try:
|
|
repo_path = os.path.join(tmpdir, 'repo')
|
|
if upstream:
|
|
run_command("git clone %(upstream)s %(repo_path)s" %
|
|
dict(upstream=upstream,
|
|
repo_path=repo_path))
|
|
git_command(repo_path,
|
|
"fetch origin "
|
|
"+refs/heads/*:refs/copy/heads/*",
|
|
env=ssh_env)
|
|
push_string = "push %s +refs/copy/heads/*:refs/heads/*"
|
|
else:
|
|
run_command("git init %s" % repo_path)
|
|
with open(os.path.join(repo_path,
|
|
".gitreview"),
|
|
'w') as gitreview:
|
|
gitreview.write("""[gerrit]
|
|
host=%s
|
|
port=29418
|
|
project=%s
|
|
""" % (GERRIT_HOST, project_git))
|
|
git_command(repo_path, "add .gitreview")
|
|
cmd = "commit -a -m'Added .gitreview' --author=" \
|
|
"'Openstack Project Creator " \
|
|
"<openstack-infra@lists.openstack.org>'"
|
|
git_command(repo_path, cmd)
|
|
push_string = "push --all %s"
|
|
gerrit.createProject(project)
|
|
|
|
if not os.path.exists(project_dir):
|
|
run_command("git --bare init %s" % project_dir)
|
|
run_command("chown -R gerrit2:gerrit2 %s"
|
|
% project_dir)
|
|
|
|
git_command(repo_path,
|
|
push_string % remote_url,
|
|
env=ssh_env)
|
|
git_command(repo_path,
|
|
"push --tags %s" % remote_url, env=ssh_env)
|
|
finally:
|
|
run_command("rm -fr %s" % tmpdir)
|
|
|
|
try:
|
|
acl_config = section.get('acl-config',
|
|
'%s.config' % os.path.join(ACL_DIR,
|
|
project))
|
|
except AttributeError:
|
|
acl_config = None
|
|
|
|
if acl_config:
|
|
if not os.path.isfile(acl_config):
|
|
write_acl_config(project,
|
|
ACL_DIR,
|
|
section.get('acl-base', None),
|
|
section.get('acl-append', []),
|
|
section.get('acl-parameters', {}))
|
|
tmpdir = tempfile.mkdtemp()
|
|
try:
|
|
repo_path = os.path.join(tmpdir, 'repo')
|
|
ret, _ = run_command_status("git init %s" % repo_path)
|
|
if ret != 0:
|
|
continue
|
|
if (fetch_config(project,
|
|
remote_url,
|
|
repo_path,
|
|
ssh_env) and
|
|
copy_acl_config(project, repo_path,
|
|
acl_config) and
|
|
create_groups_file(project, gerrit, repo_path)):
|
|
push_acl_config(project,
|
|
remote_url,
|
|
repo_path,
|
|
ssh_env)
|
|
finally:
|
|
run_command("rm -fr %s" % tmpdir)
|
|
finally:
|
|
os.unlink(ssh_env['GIT_SSH'])
|