opendev/system-config
Code Issues Proposed changes

Attempt to more fully manage project creation.

Browse Source 
Manage project creation via yaml files. Also,
Modify the manage_projects scripts to configure Gerrit project ACLs.
This change expects the project yaml to exist. The change will clone the
project for the localhost Gerrit install. It will then checkout the
meta/config ref, copy the ACL config file into the repo, commit, and
push to the origin. The ACL config location should be specified in the
projects.yaml file with the acl_config key.

For this to work the ACLs will need to be copied by Puppet from Puppet
to the Gerrit host. Add the file resource to do this as well.

Change-Id: I15a1ec13b381dce3c115c01c21f404ab79e72cc4
Reviewed-on: https://review.openstack.org/15352
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Monty Taylor <mordred@inaugust.com>
Reviewed-by: Monty Taylor <mordred@inaugust.com>
Tested-by: Jenkins
This commit is contained in:
Monty Taylor
2012-11-04 22:20:36 +01:00
committed by Jenkins
parent 85731ecb8c
commit d76f051d22
18 changed files with 810 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
modules/gerrit/files/scripts/make_local_repos.py
View File

@@ -1,65 +0,0 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Make local repos reads a project config file called projects.yaml
# It should look like:
# - project: PROJECT_NAME
# remote: https://gerrit.googlesource.com/gerrit
# TODO: add support for
# ssh -p 29418 localhost gerrit -name create-project PROJECT
import logging
import os
import subprocess
import sys
import shlex
import yaml
def run_command(cmd, status=False, env={}):
cmd_list = shlex.split(str(cmd))
newenv = os.environ
newenv.update(env)
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=newenv)
(out, nothing) = p.communicate()
if status:
return (p.returncode, out.strip())
return out.strip()
def run_command_status(cmd, env={}):
return run_command(cmd, True, env)
logging.basicConfig(level=logging.ERROR)
REPO_ROOT = sys.argv[1]
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
config = yaml.load(open(PROJECTS_YAML))
for section in config:
project = section['project']
project_git = "%s.git" % project
project_dir = os.path.join(REPO_ROOT, project_git)
if os.path.exists(project_dir):
continue
run_command("git --bare init --shared=group %s" % project_dir)

312
modules/gerrit/files/scripts/manage_projects.py Executable file
View File

@@ -0,0 +1,312 @@
#! /usr/bin/env python
# Copyright (C) 2011 OpenStack, LLC.
# Copyright (c) 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# manage_projects.py reads a project config file called projects.yaml
# It should look like:
# - homepage: http://openstack.org
# gerrit-host: review.openstack.org
# local-git-dir: /var/lib/git
# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key
# has-wiki: False
# has-issues: False
# has-downloads: False
# ---
# - project: PROJECT_NAME
# options:
# - has-wiki
# - has-issues
# - has-downloads
# - has-pull-requests
# homepage: Some homepage that isn't http://openstack.org
# description: This is a great project
# remote: https://gerrit.googlesource.com/gerrit
# upstream: git://github.com/bushy/beards.git
# acl_config: /path/to/gerrit/project.config
import ConfigParser
import logging
import os
import re
import shlex
import subprocess
import tempfile
import yaml
import github
import gerritlib.gerrit
logging.basicConfig(level=logging.ERROR)
log = logging.getLogger("manage_projects")
def run_command(cmd, status=False, env={}):
cmd_list = shlex.split(str(cmd))
newenv = os.environ
newenv.update(env)
log.debug("Executing command: %s" % " ".join(cmd_list))
p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=newenv)
(out, nothing) = p.communicate()
log.debug("Return code: %s" % p.returncode)
log.debug("Command said: %s" % out.strip())
if status:
return (p.returncode, out.strip())
return out.strip()
def run_command_status(cmd, env={}):
return run_command(cmd, True, env)
def git_command(repo_dir, sub_cmd, env={}):
git_dir = os.path.join(repo_dir, '.git')
cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd)
status, _ = run_command(cmd, True, env)
return status
def fetch_config(project, remote_url, repo_path, env={}):
status = git_command(repo_path, "fetch %s +refs/meta/config:"
"refs/remotes/gerrit-meta/config" % remote_url, env)
if status != 0:
print "Failed to fetch refs/meta/config for project: %s" % project
return False
# Because the following fails if executed more than once you should only
# run fetch_config once in each repo.
status = git_command(repo_path, "checkout -b config "
"remotes/gerrit-meta/config")
if status != 0:
print "Failed to checkout config for project: %s" % project
return False
return True
def copy_acl_config(project, repo_path, acl_config):
if not os.path.exists(acl_config):
return False
acl_dest = os.path.join(repo_path, "project.config")
status, _ = run_command("cp %s %s" %
(acl_config, acl_dest), status=True)
if status == 0:
status = git_command(repo_path, "diff-index --quiet HEAD --")
if status != 0:
return True
return False
def push_acl_config(project, remote_url, repo_path, env={}):
cmd = "commit -a -m'Update project config.' --author='Openstack Project " \
"Creator <openstack-infra@lists.openstack.org>'"
status = git_command(repo_path, cmd)
if status != 0:
print "Failed to commit config for project: %s" % project
return False
status = git_command(repo_path,
"push %s HEAD:refs/meta/config" %
remote_url, env)
if status != 0:
print "Failed to push config for project: %s" % project
return False
return True
def _get_group_uuid(gerrit, group):
query = "select group_uuid from account_groups where name = '%s'" % group
data = gerrit.dbQuery(query)
if data:
for row in data:
if row["type"] == "row":
return row["columns"]["group_uuid"]
return None
def get_group_uuid(gerrit, group):
uuid = _get_group_uuid(gerrit, group)
if uuid:
return uuid
gerrit.createGroup(group)
uuid = _get_group_uuid(gerrit, group)
if uuid:
return uuid
return None
def create_groups_file(project, gerrit, repo_path):
acl_config = os.path.join(repo_path, "project.config")
group_file = os.path.join(repo_path, "groups")
uuids = {}
for line in open(acl_config, 'r'):
r = re.match(r'^\t.*group\s(.*)$', line)
if r:
group = r.group(1)
if group in uuids.keys():
continue
uuid = get_group_uuid(gerrit, group)
if uuid:
uuids[group] = uuid
else:
return False
if uuids:
with open(group_file, 'w') as fp:
for group, uuid in uuids.items():
fp.write("%s\t%s\n" % (uuid, group))
status = git_command(repo_path, "add groups")
if status != 0:
print "Failed to add groups file for project: %s" % project
return False
return True
def make_ssh_wrapper(gerrit_user, gerrit_key):
(fd, name) = tempfile.mkstemp(text=True)
os.write(fd, '#!/bin/bash\n')
os.write(fd,
'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' %
(gerrit_key, gerrit_user))
os.close(fd)
os.chmod(name, 755)
return dict(GIT_SSH=name)
PROJECTS_YAML = os.environ.get('PROJECTS_YAML',
'/home/gerrit2/projects.yaml')
configs = [config for config in yaml.load_all(open(PROJECTS_YAML))]
defaults = configs[0][0]
default_has_issues = defaults.get('has-issues', False)
default_has_downloads = defaults.get('has-downloads', False)
default_has_wiki = defaults.get('has-wiki', False)
LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git')
GERRIT_HOST = defaults.get('gerrit-host')
GERRIT_USER = defaults.get('gerrit-user')
GERRIT_KEY = defaults.get('gerrit-key')
GITHUB_SECURE_CONFIG = defaults.get('github-config',
'/etc/github/github-projects.secure.config')
secure_config = ConfigParser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
# Project creation doesn't work via oauth
ghub = github.Github(secure_config.get("github", "username"),
secure_config.get("github", "password"))
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
gerrit = gerritlib.gerrit.Gerrit('localhost',
GERRIT_USER,
29418,
GERRIT_KEY)
project_list = gerrit.listProjects()
ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY)
try:
for section in configs[1]:
project = section['project']
options = section.get('options', dict())
description = section.get('description', None)
homepage = section.get('homepage', defaults.get('homepage', None))
upstream = section.get('upstream', None)
project_git = "%s.git" % project
project_dir = os.path.join(LOCAL_GIT_DIR, project_git)
# Find the project's repo
project_split = project.split('/', 1)
if len(project_split) > 1:
repo_name = project_split[1]
else:
repo_name = project
has_issues = 'has-issues' in options or default_has_issues
has_downloads = 'has-downloads' in options or default_has_downloads
has_wiki = 'has-wiki' in options or default_has_wiki
org = orgs_dict[project_split[0].lower()]
try:
repo = org.get_repo(repo_name)
except github.GithubException:
repo = org.create_repo(repo_name,
homepage=homepage,
has_issues=has_issues,
has_downloads=has_downloads,
has_wiki=has_wiki)
if description:
repo.edit(repo_name, description=description)
if homepage:
repo.edit(repo_name, homepage=homepage)
repo.edit(repo_name, has_issues=has_issues,
has_downloads=has_downloads,
has_wiki=has_wiki)
if 'gerrit' not in [team.name for team in repo.get_teams()]:
teams = org.get_teams()
teams_dict = dict(zip([t.name.lower() for t in teams], teams))
teams_dict['gerrit'].add_to_repos(repo)
remote_url = "ssh://localhost:29418/%s" % project
if project not in project_list:
tmpdir = tempfile.mkdtemp()
try:
repo_path = os.path.join(tmpdir, 'repo')
if upstream:
run_command("git clone %(upstream)s %(repo_path)" %
dict(upstream=upstream, repo_path=repo_path))
else:
run_command("git init %s" % repo_path)
with open(os.path.join(repo_path,
".gitreview"), 'w') as gitreview:
gitreview.write("""
[gerrit]
host=%s
port=29418
project=%s
""" % (GERRIT_HOST, project_git))
git_command(repo_path, "add .gitreview")
git_command(repo_path, "commit -a -m'Added .gitreview'")
gerrit.createProject(project)
if not os.path.exists(project_dir):
run_command("git --bare init %s" % project_dir)
run_command("chown -R gerrit2:gerrit2 %s" % project_dir)
git_command(repo_path,
"push %s HEAD:refs/heads/master" % remote_url,
env=ssh_env)
git_command(repo_path,
"push --tags %s" % remote_url, env=ssh_env)
finally:
run_command("rm -fr %s" % tmpdir)
if 'acl_config' in section:
tmpdir = tempfile.mkdtemp()
try:
repo_path = os.path.join(tmpdir, 'repo')
ret, _ = run_command_status("git init %s" % repo_path)
if ret != 0:
continue
if (fetch_config(project, remote_url, repo_path, ssh_env) and
copy_acl_config(project, repo_path,
section['acl_config']) and
create_groups_file(project, gerrit, repo_path)):
push_acl_config(project, remote_url, repo_path, ssh_env)
finally:
run_command("rm -fr %s" % tmpdir)
finally:
os.unlink(ssh_env['GIT_SSH'])

62
modules/gerrit/manifests/init.pp
View File

@@ -84,6 +84,8 @@ class gerrit(
$ssh_dsa_pubkey_contents = '', # If left empty puppet will not create file.
$ssh_rsa_key_contents = '', # If left empty puppet will not create file.
$ssh_rsa_pubkey_contents = '', # If left empty puppet will not create file.
$ssh_project_rsa_key_contents = '', # If left empty puppet will not create file.
$ssh_project_rsa_pubkey_contents = '', # If left empty puppet will not create file.
$openidssourl = 'https://login.launchpad.net/+openid',
$email = '',
$database_poollimit = '',
@@ -101,17 +103,15 @@ class gerrit(
$contactstore_appsec = '',
$contactstore_pubkey = '',
$contactstore_url = '',
$projects_file = 'UNDEF',
$enable_melody = false,
$melody_session = false,
$replicate_github = false,
$replicate_local = true,
$local_git_dir = '/var/lib/git',
$replication_targets = [],
$gitweb = true,
$testmode = false
) {
include apache
include pip
$java_home = $::lsbdistcodename ? {
'precise' => '/usr/lib/jvm/java-6-openjdk-amd64/jre',
@@ -145,6 +145,12 @@ class gerrit(
ensure => present,
}
package { 'gerritlib':
ensure => latest,
provider => 'pip',
require => Class[pip],
}
file { '/var/log/gerrit':
ensure => directory,
owner => 'gerrit2',
@@ -203,34 +209,6 @@ class gerrit(
}
}
if ($projects_file != 'UNDEF') {
if ($replicate_local) {
file { $local_git_dir:
ensure => directory,
owner => 'gerrit2',
}
}
file { '/home/gerrit2/projects.yaml':
ensure => present,
owner => 'gerrit2',
group => 'gerrit2',
mode => '0444',
source => $projects_file,
replace => true,
}
exec { 'make_local_repos':
user => 'gerrit2',
command => "/usr/local/gerrit/scripts/make_local_repos.py ${local_git_dir}",
subscribe => File['/home/gerrit2/projects.yaml'],
refreshonly => true,
require => File['/home/gerrit2/projects.yaml'],
}
}
# Gerrit sets these permissions in 'init'; don't fight them.
file { '/home/gerrit2/review_site/etc/gerrit.config':
ensure => present,
@@ -373,6 +351,28 @@ class gerrit(
}
}
if $ssh_project_rsa_key_contents != '' {
file { '/home/gerrit2/review_site/etc/ssh_project_rsa_key':
owner => 'gerrit2',
group => 'gerrit2',
mode => '0600',
content => $ssh_project_rsa_key_contents,
replace => true,
require => File['/home/gerrit2/review_site/etc']
}
}
if $ssh_project_rsa_pubkey_contents != '' {
file { '/home/gerrit2/review_site/etc/ssh_project_rsa_key.pub':
owner => 'gerrit2',
group => 'gerrit2',
mode => '0644',
content => $ssh_rsa_project_pubkey_contents,
replace => true,
require => File['/home/gerrit2/review_site/etc']
}
}
# Install Gerrit itself.
# The Gerrit WAR is specified as a url like