From d76f051d221a5662b96ad226d890c9131ba7036b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 4 Nov 2012 22:20:36 +0100 Subject: [PATCH] Attempt to more fully manage project creation. Manage project creation via yaml files. Also, Modify the manage_projects scripts to configure Gerrit project ACLs. This change expects the project yaml to exist. The change will clone the project for the localhost Gerrit install. It will then checkout the meta/config ref, copy the ACL config file into the repo, commit, and push to the origin. The ACL config location should be specified in the projects.yaml file with the acl_config key. For this to work the ACLs will need to be copied by Puppet from Puppet to the Gerrit host. Add the file resource to do this as well. Change-Id: I15a1ec13b381dce3c115c01c21f404ab79e72cc4 Reviewed-on: https://review.openstack.org/15352 Reviewed-by: Jeremy Stanley Approved: Monty Taylor Reviewed-by: Monty Taylor Tested-by: Jenkins --- doc/source/gerrit.rst | 265 ++++++++++----- manifests/site.pp | 118 ++++--- .../gerrit/files/scripts/make_local_repos.py | 65 ---- .../gerrit/files/scripts/manage_projects.py | 312 ++++++++++++++++++ modules/gerrit/manifests/init.pp | 62 ++-- .../files/scripts/close_pull_requests.py | 8 +- modules/github/manifests/init.pp | 15 + .../github-projects.secure.config.erb | 3 + .../gerrit/acls/openstack-ci-puppet.config | 18 + .../gerrit/acls/test-manage-project.config | 19 ++ .../files/review-dev.projects.yaml | 1 - modules/openstack_project/manifests/gerrit.pp | 164 ++++++--- modules/openstack_project/manifests/review.pp | 10 +- .../openstack_project/manifests/review_dev.pp | 10 +- .../templates/review-dev.projects.yaml.erb | 13 + .../review.projects.yaml.erb} | 16 + modules/pypimirror/files/run_mirror.py | 2 +- modules/pypimirror/manifests/init.pp | 4 +- 18 files changed, 810 insertions(+), 295 deletions(-) delete mode 100755 modules/gerrit/files/scripts/make_local_repos.py create mode 100755 modules/gerrit/files/scripts/manage_projects.py create mode 100644 modules/github/templates/github-projects.secure.config.erb create mode 100644 modules/openstack_project/files/gerrit/acls/openstack-ci-puppet.config create mode 100644 modules/openstack_project/files/gerrit/acls/test-manage-project.config delete mode 100644 modules/openstack_project/files/review-dev.projects.yaml create mode 100644 modules/openstack_project/templates/review-dev.projects.yaml.erb rename modules/openstack_project/{files/review.projects.yaml => templates/review.projects.yaml.erb} (81%) diff --git a/doc/source/gerrit.rst b/doc/source/gerrit.rst index 7f68e1c9e4..341d672fd4 100644 --- a/doc/source/gerrit.rst +++ b/doc/source/gerrit.rst @@ -636,31 +636,96 @@ Log into the Launchpad account and add the GPG key to the account. Adding New Projects ******************* -Creating a Project in Gerrit -============================ +Creating a new Gerrit Project with Puppet +========================================= -Using ssh key of a gerrit admin (you):: +Gerrit project creation is now managed through changes to the +openstack-ci-puppet repository. The old manual processes are documented +below as the processes are still valid and documentation of them may +still be useful when dealing with corner cases. That said, you should +use this method whenever possible. - ssh -p 29418 review.openstack.org gerrit create-project --name openstack/PROJECT +Puppet and its related scripts are able to create the new project in +Gerrit, create the new project on Github, create a local git replica on +the Gerrit host, configure the project Access Controls, and create new +groups in Gerrit that are mentioned in the Access Controls. The only +potential piece missing from this process is the management of group +membership, which is currently performed through launchpad. You might +also want to configure Zuul and Jenkins to run tests on the new project. +The details for that process are in the next section. -If the project is an API project (eg, image-api), we want it to share -some extra permissions that are common to all API projects (eg, the -OpenStack documentation coordinators can approve changes, see -:ref:`acl`). Run the following command to reparent the project if it -is an API project:: +Gerrit projects are configured in the +``openstack-ci-puppet/modules/openstack_project/templates/review.projects.yaml.erb``. +file. This file contains two sections, the first is a set of default +config values that each project can override, and the second is a list +of projects (each may contain their own overrides). - ssh -p 29418 review.openstack.org gerrit set-project-parent --parent API-Projects openstack/PROJECT +#. Config default values:: -Add yourself to the "Project Bootstrappers" group in Gerrit which will -give you permissions to push to the repo bypassing code review. + - homepage: http://example.org + local-git-dir: /var/lib/git + gerrit-host: review.example.org + gerrit-user: example-project-creator + gerrit-key: /home/gerrit2/.ssh/example_project_id_rsa + github-config: /etc/github/github-projects.secure.config + has-wiki: False + has-issues: False + has-pull-requests: False + has-downloads: False -Do the initial push of the project with:: +#. Project definition:: - git push ssh://USERNAME@review.openstack.org:29418/openstack/PROJECT.git HEAD:refs/heads/master - git push --tags ssh://USERNAME@review.openstack.org:29418/openstack/PROJECT.git + - project: example/gerrit + description: Fork of Gerrit used by Example + remote: https://gerrit.googlesource.com/gerrit + - project: example/project1 + description: Best project ever. + has-wiki: True + acl_config: /path/to/acl/file -Remove yourself from the "Project Bootstrappers" group, and then set -the access controls as specified in :ref:`acl`. +The above config gives puppet and its related scripts enough information +to create new projects, but not enough to add access controls to each +project. To add access control you need to have have an ``acl_config`` +option for the project in ``review.projects.yaml.erb`` file. That option +should have a value that is a path to the project.config for that +project. + +That is the high level view of how we can configure projects using the +pupppet repository. To create an actual change that does all of this for +a single project you will want to do the following: + +#. Add a ``modules/openstack_project/files/gerrit/acls/project-name.config`` + file to the repo. The contents will probably end up looking like:: + + [access "refs/*"] + owner = group Administrators + [receive] + requireChangeId = true + requireContributorAgreement = true + [submit] + mergeContent = true + [access "refs/heads/*"] + label-Code-Review = -2..+2 group project-name-core + label-Approved = +0..+1 group project-name-core + workInProgress = group project-name-core + [access "refs/heads/milestone-proposed"] + label-Code-Review = -2..+2 group project-name-drivers + label-Approved = +0..+1 group project-name-drivers + [project] + state = active + +You can refer to the :ref:`project-config` section below if you need +more details on writing the project.config file. + +#. Add a project entry for the project in + ``openstack-ci-puppet/modules/openstack_project/templates/review.projects.yaml.erb``.:: + + - project: openstack/project-name + acl_config: /home/gerrit2/acls/project-name.config + +That is all you need to do. Push the change to gerrit and if necessary +modify group membership for the groups you configured in the +``project.config`` through Launchpad. Have Zuul Monitor a Gerrit Project ===================================== @@ -670,6 +735,7 @@ Builder. Edit openstack/openstack-ci-puppet:modules/openstack_project/files/jenk and add the desired jobs. Most projects will use the python jobs template. A minimum config:: + - project: name: PROJECT github-org: openstack @@ -681,6 +747,7 @@ A minimum config:: - python-jobs Full example config for nova:: + - project: name: nova github-org: openstack @@ -735,6 +802,32 @@ Full example config for nova:: - nova-tarball - nova-docs +Creating a Project in Gerrit +============================ + +Using ssh key of a gerrit admin (you):: + + ssh -p 29418 review.openstack.org gerrit create-project --name openstack/PROJECT + +If the project is an API project (eg, image-api), we want it to share +some extra permissions that are common to all API projects (eg, the +OpenStack documentation coordinators can approve changes, see +:ref:`acl`). Run the following command to reparent the project if it +is an API project:: + + ssh -p 29418 review.openstack.org gerrit set-project-parent --parent API-Projects openstack/PROJECT + +Add yourself to the "Project Bootstrappers" group in Gerrit which will +give you permissions to push to the repo bypassing code review. + +Do the initial push of the project with:: + + git push ssh://USERNAME@review.openstack.org:29418/openstack/PROJECT.git HEAD:refs/heads/master + git push --tags ssh://USERNAME@review.openstack.org:29418/openstack/PROJECT.git + +Remove yourself from the "Project Bootstrappers" group, and then set +the access controls as specified in :ref:`acl`. + Create a Project in GitHub ========================== @@ -749,7 +842,7 @@ Pull requests can not be disabled for a project in Github, so instead we have a script that runs from cron to close any open pull requests with instructions to use Gerrit. -* Edit openstack/openstack-ci-puppet:modules/openstack_project/files/review.projects.yaml +* Edit openstack/openstack-ci-puppet:modules/openstack_project/templates/review.projects.yaml.erb and add the project to the list of projects in the yaml file @@ -768,6 +861,71 @@ On the gerrit host:: sudo git --bare init /var/lib/git/openstack/PROJECT.git sudo chown -R gerrit2:gerrit2 /var/lib/git/openstack/PROJECT.git +Adding A New Project On The Command Line +**************************************** + +All of the steps involved in adding a new project to Gerrit can be +accomplished via the commandline, with the exception of creating a new repo +on github. + +First of all, add the .gitreview file to the repo that will be added. Then, +assuming an ssh config alias of `review` for the gerrit instance, as a person +in the Project Bootstrappers group:: + + ssh review gerrit create-project --name openstack/$PROJECT + git review -s + git push gerrit HEAD:refs/heads/master + git push --tags gerrit + +At this point, the branch contents will be in gerrit, and the project config +settings and ACLs need to be set. These are maintained in a special branch +inside of git in gerrit. Check out the branch from git:: + + git fetch gerrit +refs/meta/*:refs/remotes/gerrit-meta/* + git checkout -b config remotes/gerrit-meta/config + +There will be two interesting files, `groups` and `project.config`. `groups` +contains UUIDs and names of groups that will be referenced in +`project.config`. There is a helper script in the openstack-ci repo called +`get_group_uuid.py` which will fetch the UUID for a given group. For +$PROJECT-core and $PROJECT-drivers:: + + openstack-ci/gerrit/get_group_uuid.py $GROUP_NAME + +And make entries in `groups` for each one of them. Next, edit +`project.config` to look like:: + + [access "refs/*"] + owner = group Administrators + [receive] + requireChangeId = true + requireContributorAgreement = true + [submit] + mergeContent = true + [access "refs/heads/*"] + label-Code-Review = -2..+2 group $PROJECT-core + label-Approved = +0..+1 group $PROJECT-core + [access "refs/heads/milestone-proposed"] + label-Code-Review = -2..+2 group $PROJECT-drivers + label-Approved = +0..+1 group $PROJECT-drivers + +If the project is for a client library, the `refs/*` section of +`project.config` should look like:: + + [access "refs/*"] + owner = group Administrators + create = group $PROJECT-drivers + pushTag = group $PROJECT-drivers + +Replace $PROJECT with the name of the project. + +Finally, commit the changes and push the config back up to Gerrit:: + + git commit -m "Initial project config" + git push gerrit HEAD:refs/meta/config + +At this point you can follow the steps above for creating the project's github +replica, the local git replica, and zuul monitoring/jenkins jobs. Migrating a Project from bzr ============================ @@ -797,8 +955,10 @@ So, for instance, to do glance, you would do: And you will then have a git repo of glance in the glance dir. This git repo is now suitable for uploading in to gerrit to become the new master repo. +.. _project-config: + Project Config -============== +************** There are a few options which need to be enabled on the project in the Admin interface. @@ -816,7 +976,7 @@ Optionally, if the PTL agrees to it: .. _acl: Access Controls -*************** +=============== High level goals: @@ -984,68 +1144,3 @@ completely delete an account from Gerrit, here's how: ssh review.openstack.org -p29418 gerrit flush-caches --all -Adding A New Project On The Command Line -**************************************** - -All of the steps involved in adding a new project to Gerrit can be -accomplished via the commandline, with the exception of creating a new repo -on github. - -First of all, add the .gitreview file to the repo that will be added. Then, -assuming an ssh config alias of `review` for the gerrit instance, as a person -in the Project Bootstrappers group:: - - ssh review gerrit create-project --name openstack/$PROJECT - git review -s - git push gerrit HEAD:refs/heads/master - git push --tags gerrit - -At this point, the branch contents will be in gerrit, and the project config -settings and ACLs need to be set. These are maintained in a special branch -inside of git in gerrit. Check out the branch from git:: - - git fetch gerrit +refs/meta/*:refs/remotes/gerrit-meta/* - git checkout -b config remotes/gerrit-meta/config - -There will be two interesting files, `groups` and `project.config`. `groups` -contains UUIDs and names of groups that will be referenced in -`project.config`. There is a helper script in the openstack-ci repo called -`get_group_uuid.py` which will fetch the UUID for a given group. For -$PROJECT-core and $PROJECT-drivers:: - - openstack-ci/gerrit/get_group_uuid.py $GROUP_NAME - -And make entries in `groups` for each one of them. Next, edit -`project.config` to look like:: - - [access "refs/*"] - owner = group Administrators - [receive] - requireChangeId = true - requireContributorAgreement = true - [submit] - mergeContent = true - [access "refs/heads/*"] - label-Code-Review = -2..+2 group $PROJECT-core - label-Approved = +0..+1 group $PROJECT-core - [access "refs/heads/milestone-proposed"] - label-Code-Review = -2..+2 group $PROJECT-drivers - label-Approved = +0..+1 group $PROJECT-drivers - -If the project is for a client library, the `refs/*` section of -`project.config` should look like:: - - [access "refs/*"] - owner = group Administrators - create = group $PROJECT-drivers - pushTag = group $PROJECT-drivers - -Replace $PROJECT with the name of the project. - -Finally, commit the changes and push the config back up to Gerrit:: - - git commit -m "Initial project config" - git push gerrit HEAD:refs/meta/config - -At this point you can follow the steps above for creating the project's github -replica, the local git replica, and zuul monitoring/jenkins jobs. diff --git a/manifests/site.pp b/manifests/site.pp index 9f4422f13f..a069c67ffa 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -13,69 +13,81 @@ node default { # node 'review.openstack.org' { class { 'openstack_project::review': - github_oauth_token => hiera('gerrit_github_token'), - mysql_password => hiera('gerrit_mysql_password'), - mysql_root_password => hiera('gerrit_mysql_root_password'), - email_private_key => hiera('gerrit_email_private_key'), - gerritbot_password => hiera('gerrit_gerritbot_password'), - ssl_cert_file_contents => hiera('gerrit_ssl_cert_file_contents'), - ssl_key_file_contents => hiera('gerrit_ssl_key_file_contents'), - ssl_chain_file_contents => hiera('gerrit_ssl_chain_file_contents'), - ssh_dsa_key_contents => hiera('gerrit_ssh_dsa_key_contents'), - ssh_dsa_pubkey_contents => hiera('gerrit_ssh_dsa_pubkey_contents'), - ssh_rsa_key_contents => hiera('gerrit_ssh_rsa_key_contents'), - ssh_rsa_pubkey_contents => hiera('gerrit_ssh_rsa_pubkey_contents'), - lp_sync_key => hiera('gerrit_lp_sync_key'), - lp_sync_pubkey => hiera('gerrit_lp_sync_pubkey'), - lp_sync_consumer_key => hiera('gerrit_lp_consumer_key'), - lp_sync_token => hiera('gerrit_lp_access_token'), - lp_sync_secret => hiera('gerrit_lp_access_secret'), - sysadmins => hiera('sysadmins'), + github_oauth_token => hiera('gerrit_github_token'), + github_project_username => hiera('github_project_username'), + github_project_password => hiera('github_project_password'), + mysql_password => hiera('gerrit_mysql_password'), + mysql_root_password => hiera('gerrit_mysql_root_password'), + email_private_key => hiera('gerrit_email_private_key'), + gerritbot_password => hiera('gerrit_gerritbot_password'), + ssl_cert_file_contents => hiera('gerrit_ssl_cert_file_contents'), + ssl_key_file_contents => hiera('gerrit_ssl_key_file_contents'), + ssl_chain_file_contents => hiera('gerrit_ssl_chain_file_contents'), + ssh_dsa_key_contents => hiera('gerrit_ssh_dsa_key_contents'), + ssh_dsa_pubkey_contents => hiera('gerrit_ssh_dsa_pubkey_contents'), + ssh_rsa_key_contents => hiera('gerrit_ssh_rsa_key_contents'), + ssh_rsa_pubkey_contents => hiera('gerrit_ssh_rsa_pubkey_contents'), + ssh_project_rsa_key_contents => hiera('gerrit_project_ssh_rsa_key_contents'), + ssh_project_rsa_pubkey_contents => hiera('gerrit_project_ssh_rsa_pubkey_contents'), + lp_sync_key => hiera('gerrit_lp_sync_key'), + lp_sync_pubkey => hiera('gerrit_lp_sync_pubkey'), + lp_sync_consumer_key => hiera('gerrit_lp_consumer_key'), + lp_sync_token => hiera('gerrit_lp_access_token'), + lp_sync_secret => hiera('gerrit_lp_access_secret'), + sysadmins => hiera('sysadmins'), } } node 'review2.openstack.org' { class { 'openstack_project::review': - github_oauth_token => hiera('gerrit_github_token'), - mysql_password => hiera('gerrit_mysql_password'), - mysql_root_password => hiera('gerrit_mysql_root_password'), - email_private_key => hiera('gerrit_email_private_key'), - gerritbot_password => hiera('gerrit_gerritbot_password'), - ssl_cert_file_contents => hiera('gerrit_ssl_cert_file_contents'), - ssl_key_file_contents => hiera('gerrit_ssl_key_file_contents'), - ssl_chain_file_contents => hiera('gerrit_ssl_chain_file_contents'), - ssh_dsa_key_contents => hiera('gerrit_ssh_dsa_key_contents'), - ssh_dsa_pubkey_contents => hiera('gerrit_ssh_dsa_pubkey_contents'), - ssh_rsa_key_contents => hiera('gerrit_ssh_rsa_key_contents'), - ssh_rsa_pubkey_contents => hiera('gerrit_ssh_rsa_pubkey_contents'), - lp_sync_key => hiera('gerrit_lp_sync_key'), - lp_sync_pubkey => hiera('gerrit_lp_sync_pubkey'), - lp_sync_consumer_key => hiera('gerrit_lp_consumer_key'), - lp_sync_token => hiera('gerrit_lp_access_token'), - lp_sync_secret => hiera('gerrit_lp_access_secret'), - replicate_github => false, - sysadmins => hiera('safesysadmins'), + github_oauth_token => hiera('gerrit_github_token'), + github_project_username => hiera('github_project_username'), + github_project_password => hiera('github_project_password'), + mysql_password => hiera('gerrit_mysql_password'), + mysql_root_password => hiera('gerrit_mysql_root_password'), + email_private_key => hiera('gerrit_email_private_key'), + gerritbot_password => hiera('gerrit_gerritbot_password'), + ssl_cert_file_contents => hiera('gerrit_ssl_cert_file_contents'), + ssl_key_file_contents => hiera('gerrit_ssl_key_file_contents'), + ssl_chain_file_contents => hiera('gerrit_ssl_chain_file_contents'), + ssh_dsa_key_contents => hiera('gerrit_ssh_dsa_key_contents'), + ssh_dsa_pubkey_contents => hiera('gerrit_ssh_dsa_pubkey_contents'), + ssh_rsa_key_contents => hiera('gerrit_ssh_rsa_key_contents'), + ssh_rsa_pubkey_contents => hiera('gerrit_ssh_rsa_pubkey_contents'), + ssh_project_rsa_key_contents => hiera('gerrit_project_ssh_rsa_key_contents'), + ssh_project_rsa_pubkey_contents => hiera('gerrit_project_ssh_rsa_pubkey_contents'), + lp_sync_key => hiera('gerrit_lp_sync_key'), + lp_sync_pubkey => hiera('gerrit_lp_sync_pubkey'), + lp_sync_consumer_key => hiera('gerrit_lp_consumer_key'), + lp_sync_token => hiera('gerrit_lp_access_token'), + lp_sync_secret => hiera('gerrit_lp_access_secret'), + replicate_github => false, + sysadmins => hiera('safesysadmins'), } } node 'review-dev.openstack.org' { class { 'openstack_project::review_dev': - github_oauth_token => hiera('gerrit_dev_github_token'), - mysql_password => hiera('gerrit_dev_mysql_password'), - mysql_root_password => hiera('gerrit_dev_mysql_root_password'), - email_private_key => hiera('gerrit_dev_email_private_key'), - contactstore_appsec => hiera('gerrit_dev_contactstore_appsec'), - contactstore_pubkey => hiera('gerrit_dev_contactstore_pubkey'), - ssh_dsa_key_contents => hiera('gerrit_dev_ssh_dsa_key_contents'), - ssh_dsa_pubkey_contents => hiera('gerrit_dev_ssh_dsa_pubkey_contents'), - ssh_rsa_key_contents => hiera('gerrit_dev_ssh_rsa_key_contents'), - ssh_rsa_pubkey_contents => hiera('gerrit_dev_ssh_rsa_pubkey_contents'), - lp_sync_key => hiera('gerrit_dev_lp_sync_key'), - lp_sync_pubkey => hiera('gerrit_dev_lp_sync_pubkey'), - lp_sync_consumer_key => hiera('gerrit_dev_lp_consumer_key'), - lp_sync_token => hiera('gerrit_dev_lp_access_token'), - lp_sync_secret => hiera('gerrit_dev_lp_access_secret'), - sysadmins => hiera('sysadmins'), + github_oauth_token => hiera('gerrit_dev_github_token'), + github_project_username => hiera('github_dev_project_username'), + github_project_password => hiera('github_dev_project_password'), + mysql_password => hiera('gerrit_dev_mysql_password'), + mysql_root_password => hiera('gerrit_dev_mysql_root_password'), + email_private_key => hiera('gerrit_dev_email_private_key'), + contactstore_appsec => hiera('gerrit_dev_contactstore_appsec'), + contactstore_pubkey => hiera('gerrit_dev_contactstore_pubkey'), + ssh_dsa_key_contents => hiera('gerrit_dev_ssh_dsa_key_contents'), + ssh_dsa_pubkey_contents => hiera('gerrit_dev_ssh_dsa_pubkey_contents'), + ssh_rsa_key_contents => hiera('gerrit_dev_ssh_rsa_key_contents'), + ssh_rsa_pubkey_contents => hiera('gerrit_dev_ssh_rsa_pubkey_contents'), + ssh_project_rsa_key_contents => hiera('gerrit_dev_project_ssh_rsa_key_contents'), + ssh_project_rsa_pubkey_contents => hiera('gerrit_dev_project_ssh_rsa_pubkey_contents'), + lp_sync_key => hiera('gerrit_dev_lp_sync_key'), + lp_sync_pubkey => hiera('gerrit_dev_lp_sync_pubkey'), + lp_sync_consumer_key => hiera('gerrit_dev_lp_consumer_key'), + lp_sync_token => hiera('gerrit_dev_lp_access_token'), + lp_sync_secret => hiera('gerrit_dev_lp_access_secret'), + sysadmins => hiera('sysadmins'), } } diff --git a/modules/gerrit/files/scripts/make_local_repos.py b/modules/gerrit/files/scripts/make_local_repos.py deleted file mode 100755 index bb8ab9d899..0000000000 --- a/modules/gerrit/files/scripts/make_local_repos.py +++ /dev/null @@ -1,65 +0,0 @@ -#! /usr/bin/env python -# Copyright (C) 2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Make local repos reads a project config file called projects.yaml -# It should look like: - -# - project: PROJECT_NAME -# remote: https://gerrit.googlesource.com/gerrit - -# TODO: add support for -# ssh -p 29418 localhost gerrit -name create-project PROJECT - -import logging -import os -import subprocess -import sys -import shlex -import yaml - -def run_command(cmd, status=False, env={}): - cmd_list = shlex.split(str(cmd)) - newenv = os.environ - newenv.update(env) - p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=newenv) - (out, nothing) = p.communicate() - if status: - return (p.returncode, out.strip()) - return out.strip() - - -def run_command_status(cmd, env={}): - return run_command(cmd, True, env) - - -logging.basicConfig(level=logging.ERROR) - -REPO_ROOT = sys.argv[1] -PROJECTS_YAML = os.environ.get('PROJECTS_YAML', - '/home/gerrit2/projects.yaml') - -config = yaml.load(open(PROJECTS_YAML)) - -for section in config: - project = section['project'] - - project_git = "%s.git" % project - project_dir = os.path.join(REPO_ROOT, project_git) - - if os.path.exists(project_dir): - continue - - run_command("git --bare init --shared=group %s" % project_dir) diff --git a/modules/gerrit/files/scripts/manage_projects.py b/modules/gerrit/files/scripts/manage_projects.py new file mode 100755 index 0000000000..fb50b26e76 --- /dev/null +++ b/modules/gerrit/files/scripts/manage_projects.py @@ -0,0 +1,312 @@ +#! /usr/bin/env python +# Copyright (C) 2011 OpenStack, LLC. +# Copyright (c) 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# manage_projects.py reads a project config file called projects.yaml +# It should look like: + +# - homepage: http://openstack.org +# gerrit-host: review.openstack.org +# local-git-dir: /var/lib/git +# gerrit-key: /home/gerrit2/review_site/etc/ssh_host_rsa_key +# has-wiki: False +# has-issues: False +# has-downloads: False +# --- +# - project: PROJECT_NAME +# options: +# - has-wiki +# - has-issues +# - has-downloads +# - has-pull-requests +# homepage: Some homepage that isn't http://openstack.org +# description: This is a great project +# remote: https://gerrit.googlesource.com/gerrit +# upstream: git://github.com/bushy/beards.git +# acl_config: /path/to/gerrit/project.config + + +import ConfigParser +import logging +import os +import re +import shlex +import subprocess +import tempfile +import yaml + +import github +import gerritlib.gerrit + + +logging.basicConfig(level=logging.ERROR) +log = logging.getLogger("manage_projects") + + +def run_command(cmd, status=False, env={}): + cmd_list = shlex.split(str(cmd)) + newenv = os.environ + newenv.update(env) + log.debug("Executing command: %s" % " ".join(cmd_list)) + p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=newenv) + (out, nothing) = p.communicate() + log.debug("Return code: %s" % p.returncode) + log.debug("Command said: %s" % out.strip()) + if status: + return (p.returncode, out.strip()) + return out.strip() + + +def run_command_status(cmd, env={}): + return run_command(cmd, True, env) + + +def git_command(repo_dir, sub_cmd, env={}): + git_dir = os.path.join(repo_dir, '.git') + cmd = "git --git-dir=%s --work-tree=%s %s" % (git_dir, repo_dir, sub_cmd) + status, _ = run_command(cmd, True, env) + return status + + +def fetch_config(project, remote_url, repo_path, env={}): + status = git_command(repo_path, "fetch %s +refs/meta/config:" + "refs/remotes/gerrit-meta/config" % remote_url, env) + if status != 0: + print "Failed to fetch refs/meta/config for project: %s" % project + return False + # Because the following fails if executed more than once you should only + # run fetch_config once in each repo. + status = git_command(repo_path, "checkout -b config " + "remotes/gerrit-meta/config") + if status != 0: + print "Failed to checkout config for project: %s" % project + return False + + return True + + +def copy_acl_config(project, repo_path, acl_config): + if not os.path.exists(acl_config): + return False + + acl_dest = os.path.join(repo_path, "project.config") + status, _ = run_command("cp %s %s" % + (acl_config, acl_dest), status=True) + if status == 0: + status = git_command(repo_path, "diff-index --quiet HEAD --") + if status != 0: + return True + return False + + +def push_acl_config(project, remote_url, repo_path, env={}): + cmd = "commit -a -m'Update project config.' --author='Openstack Project " \ + "Creator '" + status = git_command(repo_path, cmd) + if status != 0: + print "Failed to commit config for project: %s" % project + return False + status = git_command(repo_path, + "push %s HEAD:refs/meta/config" % + remote_url, env) + if status != 0: + print "Failed to push config for project: %s" % project + return False + return True + + +def _get_group_uuid(gerrit, group): + query = "select group_uuid from account_groups where name = '%s'" % group + data = gerrit.dbQuery(query) + if data: + for row in data: + if row["type"] == "row": + return row["columns"]["group_uuid"] + return None + + +def get_group_uuid(gerrit, group): + uuid = _get_group_uuid(gerrit, group) + if uuid: + return uuid + gerrit.createGroup(group) + uuid = _get_group_uuid(gerrit, group) + if uuid: + return uuid + return None + + +def create_groups_file(project, gerrit, repo_path): + acl_config = os.path.join(repo_path, "project.config") + group_file = os.path.join(repo_path, "groups") + uuids = {} + for line in open(acl_config, 'r'): + r = re.match(r'^\t.*group\s(.*)$', line) + if r: + group = r.group(1) + if group in uuids.keys(): + continue + uuid = get_group_uuid(gerrit, group) + if uuid: + uuids[group] = uuid + else: + return False + if uuids: + with open(group_file, 'w') as fp: + for group, uuid in uuids.items(): + fp.write("%s\t%s\n" % (uuid, group)) + status = git_command(repo_path, "add groups") + if status != 0: + print "Failed to add groups file for project: %s" % project + return False + return True + + +def make_ssh_wrapper(gerrit_user, gerrit_key): + (fd, name) = tempfile.mkstemp(text=True) + os.write(fd, '#!/bin/bash\n') + os.write(fd, + 'ssh -i %s -l %s -o "StrictHostKeyChecking no" $@\n' % + (gerrit_key, gerrit_user)) + os.close(fd) + os.chmod(name, 755) + return dict(GIT_SSH=name) + + +PROJECTS_YAML = os.environ.get('PROJECTS_YAML', + '/home/gerrit2/projects.yaml') +configs = [config for config in yaml.load_all(open(PROJECTS_YAML))] +defaults = configs[0][0] +default_has_issues = defaults.get('has-issues', False) +default_has_downloads = defaults.get('has-downloads', False) +default_has_wiki = defaults.get('has-wiki', False) + +LOCAL_GIT_DIR = defaults.get('local-git-dir', '/var/lib/git') +GERRIT_HOST = defaults.get('gerrit-host') +GERRIT_USER = defaults.get('gerrit-user') +GERRIT_KEY = defaults.get('gerrit-key') +GITHUB_SECURE_CONFIG = defaults.get('github-config', + '/etc/github/github-projects.secure.config') + +secure_config = ConfigParser.ConfigParser() +secure_config.read(GITHUB_SECURE_CONFIG) + +# Project creation doesn't work via oauth +ghub = github.Github(secure_config.get("github", "username"), + secure_config.get("github", "password")) +orgs = ghub.get_user().get_orgs() +orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) + +gerrit = gerritlib.gerrit.Gerrit('localhost', + GERRIT_USER, + 29418, + GERRIT_KEY) +project_list = gerrit.listProjects() +ssh_env = make_ssh_wrapper(GERRIT_USER, GERRIT_KEY) +try: + + for section in configs[1]: + project = section['project'] + options = section.get('options', dict()) + description = section.get('description', None) + homepage = section.get('homepage', defaults.get('homepage', None)) + upstream = section.get('upstream', None) + + project_git = "%s.git" % project + project_dir = os.path.join(LOCAL_GIT_DIR, project_git) + + # Find the project's repo + project_split = project.split('/', 1) + if len(project_split) > 1: + repo_name = project_split[1] + else: + repo_name = project + has_issues = 'has-issues' in options or default_has_issues + has_downloads = 'has-downloads' in options or default_has_downloads + has_wiki = 'has-wiki' in options or default_has_wiki + org = orgs_dict[project_split[0].lower()] + try: + repo = org.get_repo(repo_name) + except github.GithubException: + repo = org.create_repo(repo_name, + homepage=homepage, + has_issues=has_issues, + has_downloads=has_downloads, + has_wiki=has_wiki) + if description: + repo.edit(repo_name, description=description) + if homepage: + repo.edit(repo_name, homepage=homepage) + + repo.edit(repo_name, has_issues=has_issues, + has_downloads=has_downloads, + has_wiki=has_wiki) + + if 'gerrit' not in [team.name for team in repo.get_teams()]: + teams = org.get_teams() + teams_dict = dict(zip([t.name.lower() for t in teams], teams)) + teams_dict['gerrit'].add_to_repos(repo) + + remote_url = "ssh://localhost:29418/%s" % project + if project not in project_list: + tmpdir = tempfile.mkdtemp() + try: + repo_path = os.path.join(tmpdir, 'repo') + if upstream: + run_command("git clone %(upstream)s %(repo_path)" % + dict(upstream=upstream, repo_path=repo_path)) + else: + run_command("git init %s" % repo_path) + with open(os.path.join(repo_path, + ".gitreview"), 'w') as gitreview: + gitreview.write(""" +[gerrit] +host=%s +port=29418 +project=%s +""" % (GERRIT_HOST, project_git)) + git_command(repo_path, "add .gitreview") + git_command(repo_path, "commit -a -m'Added .gitreview'") + gerrit.createProject(project) + + if not os.path.exists(project_dir): + run_command("git --bare init %s" % project_dir) + run_command("chown -R gerrit2:gerrit2 %s" % project_dir) + + git_command(repo_path, + "push %s HEAD:refs/heads/master" % remote_url, + env=ssh_env) + git_command(repo_path, + "push --tags %s" % remote_url, env=ssh_env) + finally: + run_command("rm -fr %s" % tmpdir) + + if 'acl_config' in section: + tmpdir = tempfile.mkdtemp() + try: + repo_path = os.path.join(tmpdir, 'repo') + ret, _ = run_command_status("git init %s" % repo_path) + if ret != 0: + continue + if (fetch_config(project, remote_url, repo_path, ssh_env) and + copy_acl_config(project, repo_path, + section['acl_config']) and + create_groups_file(project, gerrit, repo_path)): + push_acl_config(project, remote_url, repo_path, ssh_env) + finally: + run_command("rm -fr %s" % tmpdir) +finally: + os.unlink(ssh_env['GIT_SSH']) diff --git a/modules/gerrit/manifests/init.pp b/modules/gerrit/manifests/init.pp index 070e607b96..7cf341b736 100644 --- a/modules/gerrit/manifests/init.pp +++ b/modules/gerrit/manifests/init.pp @@ -84,6 +84,8 @@ class gerrit( $ssh_dsa_pubkey_contents = '', # If left empty puppet will not create file. $ssh_rsa_key_contents = '', # If left empty puppet will not create file. $ssh_rsa_pubkey_contents = '', # If left empty puppet will not create file. + $ssh_project_rsa_key_contents = '', # If left empty puppet will not create file. + $ssh_project_rsa_pubkey_contents = '', # If left empty puppet will not create file. $openidssourl = 'https://login.launchpad.net/+openid', $email = '', $database_poollimit = '', @@ -101,17 +103,15 @@ class gerrit( $contactstore_appsec = '', $contactstore_pubkey = '', $contactstore_url = '', - $projects_file = 'UNDEF', $enable_melody = false, $melody_session = false, $replicate_github = false, - $replicate_local = true, - $local_git_dir = '/var/lib/git', $replication_targets = [], $gitweb = true, $testmode = false ) { include apache + include pip $java_home = $::lsbdistcodename ? { 'precise' => '/usr/lib/jvm/java-6-openjdk-amd64/jre', @@ -145,6 +145,12 @@ class gerrit( ensure => present, } + package { 'gerritlib': + ensure => latest, + provider => 'pip', + require => Class[pip], + } + file { '/var/log/gerrit': ensure => directory, owner => 'gerrit2', @@ -203,34 +209,6 @@ class gerrit( } } - if ($projects_file != 'UNDEF') { - - if ($replicate_local) { - file { $local_git_dir: - ensure => directory, - owner => 'gerrit2', - } - } - - file { '/home/gerrit2/projects.yaml': - ensure => present, - owner => 'gerrit2', - group => 'gerrit2', - mode => '0444', - source => $projects_file, - replace => true, - } - - exec { 'make_local_repos': - user => 'gerrit2', - command => "/usr/local/gerrit/scripts/make_local_repos.py ${local_git_dir}", - subscribe => File['/home/gerrit2/projects.yaml'], - refreshonly => true, - require => File['/home/gerrit2/projects.yaml'], - } - - } - # Gerrit sets these permissions in 'init'; don't fight them. file { '/home/gerrit2/review_site/etc/gerrit.config': ensure => present, @@ -373,6 +351,28 @@ class gerrit( } } + if $ssh_project_rsa_key_contents != '' { + file { '/home/gerrit2/review_site/etc/ssh_project_rsa_key': + owner => 'gerrit2', + group => 'gerrit2', + mode => '0600', + content => $ssh_project_rsa_key_contents, + replace => true, + require => File['/home/gerrit2/review_site/etc'] + } + } + + if $ssh_project_rsa_pubkey_contents != '' { + file { '/home/gerrit2/review_site/etc/ssh_project_rsa_key.pub': + owner => 'gerrit2', + group => 'gerrit2', + mode => '0644', + content => $ssh_rsa_project_pubkey_contents, + replace => true, + require => File['/home/gerrit2/review_site/etc'] + } + } + # Install Gerrit itself. # The Gerrit WAR is specified as a url like diff --git a/modules/github/files/scripts/close_pull_requests.py b/modules/github/files/scripts/close_pull_requests.py index 441aab95cb..cecbd324cc 100755 --- a/modules/github/files/scripts/close_pull_requests.py +++ b/modules/github/files/scripts/close_pull_requests.py @@ -16,6 +16,12 @@ # Github pull requests closer reads a project config file called projects.yaml # It should look like: +# - homepage: http://openstack.org +# team-id: 153703 +# has-wiki: False +# has-issues: False +# has-downloads: False +# --- # - project: PROJECT_NAME # options: # - has-pull-requests @@ -54,7 +60,7 @@ Please visit http://wiki.openstack.org/GerritWorkflow and follow the instruction secure_config = ConfigParser.ConfigParser() secure_config.read(GITHUB_SECURE_CONFIG) -config = yaml.load(open(PROJECTS_YAML)) +(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))] if secure_config.has_option("github", "oauth_token"): ghub = github.Github(secure_config.get("github", "oauth_token")) diff --git a/modules/github/manifests/init.pp b/modules/github/manifests/init.pp index c04f75b896..ee56ce4219 100644 --- a/modules/github/manifests/init.pp +++ b/modules/github/manifests/init.pp @@ -1,6 +1,8 @@ class github( $username, $oauth_token, + $project_username, + $project_password, $projects = [] ) { include pip @@ -55,6 +57,19 @@ class github( ], } + file { '/etc/github/github-projects.secure.config': + ensure => present, + content => template('github/github-projects.secure.config.erb'), + group => 'github', + mode => '0440', + owner => 'root', + replace => true, + require => [ + Group['github'], + File['/etc/github'] + ], + } + file { '/usr/local/github': ensure => directory, group => 'root', diff --git a/modules/github/templates/github-projects.secure.config.erb b/modules/github/templates/github-projects.secure.config.erb new file mode 100644 index 0000000000..887c2a7038 --- /dev/null +++ b/modules/github/templates/github-projects.secure.config.erb @@ -0,0 +1,3 @@ +[github] +username = <%= project_username %> +password = <%= project_password %> diff --git a/modules/openstack_project/files/gerrit/acls/openstack-ci-puppet.config b/modules/openstack_project/files/gerrit/acls/openstack-ci-puppet.config new file mode 100644 index 0000000000..2fefc9800d --- /dev/null +++ b/modules/openstack_project/files/gerrit/acls/openstack-ci-puppet.config @@ -0,0 +1,18 @@ +[access "refs/*"] + owner = group Administrators +[access "refs/heads/*"] + label-Code-Review = -2..+2 group openstack-ci-admins + label-Approved = +0..+1 group openstack-ci-admins +[receive] + requireChangeId = true + requireShortMessage = true +[submit] + mergeContent = true + action = cherry pick +[project] + state = active +[access "refs/meta/config"] + label-Approved = +0..+1 group openstack-ci-admins + label-Code-Review = -2..+2 group openstack-ci-admins + label-Verified = -2..+2 group openstack-ci-admins + submit = group openstack-ci-admins diff --git a/modules/openstack_project/files/gerrit/acls/test-manage-project.config b/modules/openstack_project/files/gerrit/acls/test-manage-project.config new file mode 100644 index 0000000000..69c3ff0333 --- /dev/null +++ b/modules/openstack_project/files/gerrit/acls/test-manage-project.config @@ -0,0 +1,19 @@ +[access "refs/*"] + owner = group Administrators +[access "refs/heads/*"] + label-Code-Review = -2..+2 group openstack-ci-admins + label-Approved = +0..+1 group openstack-ci-admins + workInProgress = group openstack-ci-admins +[receive] + requireChangeId = true + requireShortMessage = true +[submit] + mergeContent = true + action = cherry pick +[project] + state = active +[access "refs/meta/config"] + label-Approved = +0..+1 group openstack-ci-admins + label-Code-Review = -2..+2 group openstack-ci-admins + label-Verified = -2..+2 group openstack-ci-admins + submit = group openstack-ci-admins diff --git a/modules/openstack_project/files/review-dev.projects.yaml b/modules/openstack_project/files/review-dev.projects.yaml deleted file mode 100644 index c7689b088e..0000000000 --- a/modules/openstack_project/files/review-dev.projects.yaml +++ /dev/null @@ -1 +0,0 @@ -- project: gtest-org/test diff --git a/modules/openstack_project/manifests/gerrit.pp b/modules/openstack_project/manifests/gerrit.pp index 4980804954..490905ba3f 100644 --- a/modules/openstack_project/manifests/gerrit.pp +++ b/modules/openstack_project/manifests/gerrit.pp @@ -8,6 +8,7 @@ class openstack_project::gerrit ( $canonicalweburl="https://$fqdn/", $serveradmin='webmaster@openstack.org', $ssh_host_key='/home/gerrit2/review_site/etc/ssh_host_rsa_key', + $ssh_project_key='/home/gerrit2/review_site/etc/ssh_project_rsa_key', $ssl_cert_file='', $ssl_key_file='', $ssl_chain_file='', @@ -18,6 +19,8 @@ class openstack_project::gerrit ( $ssh_dsa_pubkey_contents='', # If left empty puppet will not create file. $ssh_rsa_key_contents='', # If left empty puppet will not create file. $ssh_rsa_pubkey_contents='', # If left empty puppet will not create file. + $ssh_project_rsa_key_contents='', # If left empty puppet will not create file. + $ssh_project_rsa_pubkey_contents='', # If left empty puppet will not create file. $email='', $database_poollimit='', $container_heaplimit='', @@ -40,11 +43,15 @@ class openstack_project::gerrit ( $projects_file='UNDEF', $github_username, $github_oauth_token, + $github_project_username, + $github_project_password, $mysql_password, $mysql_root_password, $trivial_rebase_role_id, $email_private_key, $replicate_github=true, + $replicate_local=true, + $local_git_dir='/var/lib/git', $testmode=false, $sysadmins=[] ) { @@ -54,69 +61,72 @@ class openstack_project::gerrit ( } class { '::gerrit': - vhost_name => $vhost_name, - canonicalweburl => $canonicalweburl, + vhost_name => $vhost_name, + canonicalweburl => $canonicalweburl, # opinions - enable_melody => 'true', - melody_session => 'true', + enable_melody => 'true', + melody_session => 'true', # passthrough - ssl_cert_file => $ssl_cert_file, - ssl_key_file => $ssl_key_file, - ssl_chain_file => $ssl_chain_file, - ssl_cert_file_contents => $ssl_cert_file_contents, - ssl_key_file_contents => $ssl_key_file_contents, - ssl_chain_file_contents => $ssl_chain_file_contents, - ssh_dsa_key_contents => $ssh_dsa_key_contents, - ssh_dsa_pubkey_contents => $ssh_dsa_pubkey_contents, - ssh_rsa_key_contents => $ssh_rsa_key_contents, - ssh_rsa_pubkey_contents => $ssh_rsa_pubkey_contents, - email => $email, - openidssourl => "https://login.launchpad.net/+openid", - database_poollimit => $database_poollimit, - container_heaplimit => $container_heaplimit, - core_packedgitopenfiles => $core_packedgitopenfiles, - core_packedgitlimit => $core_packedgitlimit, - core_packedgitwindowsize => $core_packedgitwindowsize, - sshd_threads => $sshd_threads, - httpd_acceptorthreads => $httpd_acceptorthreads, - httpd_minthreads => $httpd_minthreads, - httpd_maxthreads => $httpd_maxthreads, - httpd_maxwait => $httpd_maxwait, - commentlinks => [{ name => 'changeid', - match => '(I[0-9a-f]{8,40})', - link => '#q,$1,n,z' - }, - { name => 'launchpad', - match => '([Bb]ug|[Ll][Pp])[\\s#:]*(\\d+)', - link => 'https://code.launchpad.net/bugs/$2' - }, - { name => 'blueprint', - match => '([Bb]lue[Pp]rint|[Bb][Pp])[\\s#:]*([A-Za-z0-9\\-]+)', - link => 'https://blueprints.launchpad.net/openstack/?searchtext=$2' - }, - ], - war => $war, - contactstore => $contactstore, - contactstore_appsec => $contactstore_appsec, - contactstore_pubkey => $contactstore_pubkey, - contactstore_url => $contactstore_url, - mysql_password => $mysql_password, - mysql_root_password => $mysql_root_password, - email_private_key => $email_private_key, - projects_file => $projects_file, - replicate_github => $replicate_github, - testmode => $testmode, - require => Class[openstack_project::server], + ssl_cert_file => $ssl_cert_file, + ssl_key_file => $ssl_key_file, + ssl_chain_file => $ssl_chain_file, + ssl_cert_file_contents => $ssl_cert_file_contents, + ssl_key_file_contents => $ssl_key_file_contents, + ssl_chain_file_contents => $ssl_chain_file_contents, + ssh_dsa_key_contents => $ssh_dsa_key_contents, + ssh_dsa_pubkey_contents => $ssh_dsa_pubkey_contents, + ssh_rsa_key_contents => $ssh_rsa_key_contents, + ssh_rsa_pubkey_contents => $ssh_rsa_pubkey_contents, + ssh_project_rsa_key_contents => $ssh_project_rsa_key_contents, + ssh_project_rsa_pubkey_contents => $ssh_project_rsa_pubkey_contents, + email => $email, + openidssourl => "https://login.launchpad.net/+openid", + database_poollimit => $database_poollimit, + container_heaplimit => $container_heaplimit, + core_packedgitopenfiles => $core_packedgitopenfiles, + core_packedgitlimit => $core_packedgitlimit, + core_packedgitwindowsize => $core_packedgitwindowsize, + sshd_threads => $sshd_threads, + httpd_acceptorthreads => $httpd_acceptorthreads, + httpd_minthreads => $httpd_minthreads, + httpd_maxthreads => $httpd_maxthreads, + httpd_maxwait => $httpd_maxwait, + commentlinks => [{ name => 'changeid', + match => '(I[0-9a-f]{8,40})', + link => '#q,$1,n,z' + }, + { name => 'launchpad', + match => '([Bb]ug|[Ll][Pp])[\\s#:]*(\\d+)', + link => 'https://code.launchpad.net/bugs/$2' + }, + { name => 'blueprint', + match => '([Bb]lue[Pp]rint|[Bb][Pp])[\\s#:]*([A-Za-z0-9\\-]+)', + link => 'https://blueprints.launchpad.net/openstack/?searchtext=$2' + }, + ], + war => $war, + contactstore => $contactstore, + contactstore_appsec => $contactstore_appsec, + contactstore_pubkey => $contactstore_pubkey, + contactstore_url => $contactstore_url, + mysql_password => $mysql_password, + mysql_root_password => $mysql_root_password, + email_private_key => $email_private_key, + replicate_github => $replicate_github, + testmode => $testmode, + require => Class[openstack_project::server], } if ($testmode == false) { class { 'gerrit::cron': - script_user => $script_user, + script_user => $script_user, script_key_file => $script_key_file, } class { 'github': - username => $github_username, - oauth_token => $github_oauth_token, - require => Class['::gerrit'] + username => $github_username, + project_username => $github_project_username, + project_password => $github_project_password, + oauth_token => $github_oauth_token, + require => Class['::gerrit'] } } @@ -207,4 +217,48 @@ class openstack_project::gerrit ( replace => 'true', require => Class['::gerrit'] } + + if ($projects_file != 'UNDEF') { + if ($replicate_local) { + file { $local_git_dir: + ensure => directory, + owner => 'gerrit2', + require => Class['::gerrit'], + } + } + + file { '/home/gerrit2/projects.yaml': + ensure => present, + owner => 'gerrit2', + group => 'gerrit2', + mode => '0444', + content => template($projects_file), + replace => true, + require => Class['::gerrit'], + } + + file { '/home/gerrit2/acls': + ensure => directory, + owner => 'gerrit2', + group => 'gerrit2', + mode => '0444', + recurse => true, + replace => true, + source => 'puppet:///modules/openstack_project/gerrit/acls', + require => Class['::gerrit'] + } + + exec { 'manage_projects': + command => '/usr/local/gerrit/scripts/manage_projects.py', + subscribe => [ + File['/home/gerrit2/projects.yaml'], + File['/home/gerrit2/acls'], + ], + refreshonly => true, + require => [ + File['/home/gerrit2/projects.yaml'], + File['/home/gerrit2/acls'], + ], + } + } } diff --git a/modules/openstack_project/manifests/review.pp b/modules/openstack_project/manifests/review.pp index b1aae1c5b8..0ebe3086ef 100644 --- a/modules/openstack_project/manifests/review.pp +++ b/modules/openstack_project/manifests/review.pp @@ -26,6 +26,8 @@ # thus, set it to 5000minutes until the bug is fixed. class openstack_project::review ( $github_oauth_token, + $github_project_username, + $github_project_password, $mysql_password, $mysql_root_password, $email_private_key, @@ -37,6 +39,8 @@ class openstack_project::review ( $ssh_dsa_pubkey_contents='', $ssh_rsa_key_contents='', $ssh_rsa_pubkey_contents='', + $ssh_project_rsa_key_contents='', + $ssh_project_rsa_pubkey_contents='', $lp_sync_key='', # If left empty puppet will not create file. $lp_sync_pubkey='', # If left empty puppet will not create file. $lp_sync_consumer_key='', @@ -56,6 +60,8 @@ class openstack_project::review ( ssh_dsa_pubkey_contents => $ssh_dsa_pubkey_contents, ssh_rsa_key_contents => $ssh_rsa_key_contents, ssh_rsa_pubkey_contents => $ssh_rsa_pubkey_contents, + ssh_project_rsa_key_contents => $ssh_project_rsa_key_contents, + ssh_project_rsa_pubkey_contents => $ssh_project_rsa_pubkey_contents, email => 'review@openstack.org', database_poollimit => '150', # 1 + 100 + 9 + 2 + 2 + 25 = 139(rounded up) container_heaplimit => '8g', @@ -68,9 +74,11 @@ class openstack_project::review ( script_user => 'launchpadsync', script_key_file => '/home/gerrit2/.ssh/launchpadsync_rsa', script_logging_conf => '/home/gerrit2/.sync_logging.conf', - projects_file => 'puppet:///openstack_project/review.projects.yaml', + projects_file => 'openstack_project/review.projects.yaml.erb', github_username => 'openstack-gerrit', github_oauth_token => $github_oauth_token, + github_project_username => $github_project_username, + github_project_password => $github_project_password, mysql_password => $mysql_password, mysql_root_password => $mysql_root_password, trivial_rebase_role_id => 'trivial-rebase@review.openstack.org', diff --git a/modules/openstack_project/manifests/review_dev.pp b/modules/openstack_project/manifests/review_dev.pp index 341df2dbf3..c82c70a7cf 100644 --- a/modules/openstack_project/manifests/review_dev.pp +++ b/modules/openstack_project/manifests/review_dev.pp @@ -1,5 +1,7 @@ class openstack_project::review_dev ( $github_oauth_token, + $github_project_username, + $github_project_password, $mysql_password, $mysql_root_password, $email_private_key, @@ -9,6 +11,8 @@ class openstack_project::review_dev ( $ssh_dsa_pubkey_contents='', $ssh_rsa_key_contents='', $ssh_rsa_pubkey_contents='', + $ssh_project_rsa_key_contents='', + $ssh_project_rsa_pubkey_contents='', $cla_description='OpenStack Individual Contributor License Agreement', $cla_file='static/cla.html', $cla_id='2', @@ -31,6 +35,8 @@ class openstack_project::review_dev ( ssh_dsa_pubkey_contents => $ssh_dsa_pubkey_contents, ssh_rsa_key_contents => $ssh_rsa_key_contents, ssh_rsa_pubkey_contents => $ssh_rsa_pubkey_contents, + ssh_project_rsa_key_contents => $ssh_project_rsa_key_contents, + ssh_project_rsa_pubkey_contents => $ssh_project_rsa_pubkey_contents, email => "review-dev@openstack.org", war => 'http://tarballs.openstack.org/ci/test/gerrit-2.4.2-14-gd77b4cd.war', contactstore => true, @@ -40,9 +46,11 @@ class openstack_project::review_dev ( script_user => 'launchpadsync', script_key_file => '/home/gerrit2/.ssh/launchpadsync_rsa', script_logging_conf => '/home/gerrit2/.sync_logging.conf', - projects_file => 'puppet:///openstack_project/review-dev.projects.yaml', + projects_file => 'openstack_project/review-dev.projects.yaml.erb', github_username => 'openstack-gerrit-dev', github_oauth_token => $github_oauth_token, + github_project_username => $github_project_username, + github_project_password => $github_project_password, mysql_password => $mysql_password, mysql_root_password => $mysql_root_password, trivial_rebase_role_id => 'trivial-rebase@review-dev.openstack.org', diff --git a/modules/openstack_project/templates/review-dev.projects.yaml.erb b/modules/openstack_project/templates/review-dev.projects.yaml.erb new file mode 100644 index 0000000000..5660db6fa4 --- /dev/null +++ b/modules/openstack_project/templates/review-dev.projects.yaml.erb @@ -0,0 +1,13 @@ +- homepage: http://openstack.org + local-git-dir: <%= local_git_dir %> + gerrit-host: <%= fqdn %> + gerrit-user: openstack-dev-project-creator + gerrit-key: <%= ssh_project_key %> + github-config: /etc/github/github-projects.secure.config + has-wiki: False + has-issues: False + has-downloads: False +--- +- project: gtest-org/gtest +- project: gtest-org/test-manage-project + acl_config: /home/gerrit2/acls/test-manage-project.config diff --git a/modules/openstack_project/files/review.projects.yaml b/modules/openstack_project/templates/review.projects.yaml.erb similarity index 81% rename from modules/openstack_project/files/review.projects.yaml rename to modules/openstack_project/templates/review.projects.yaml.erb index 31344006de..82a9fd5bb3 100644 --- a/modules/openstack_project/files/review.projects.yaml +++ b/modules/openstack_project/templates/review.projects.yaml.erb @@ -1,7 +1,22 @@ +- homepage: http://openstack.org + local-git-dir: <%= local_git_dir %> + gerrit-host: <%= fqdn %> + gerrit-user: openstack-project-creator + gerrit-key: <%= ssh_project_key %> + github-config: /etc/github/github-projects.secure.config + has-wiki: False + has-issues: False + has-pull-requests: False + has-downloads: False +--- - project: heat-api/heat + options: + - has-wiki + description: '"Heat" keeps the clouds up ;)' - project: heat-api/python-heatclient - project: openstack-ci/devstack-gate - project: openstack-ci/gerrit + description: Fork of Gerrit used by OpenStack remote: https://gerrit.googlesource.com/gerrit - project: openstack-ci/gerrit-verification-status-plugin - project: openstack-ci/gerritbot @@ -36,6 +51,7 @@ - project: openstack/openstack-chef - project: openstack/openstack-ci - project: openstack/openstack-ci-puppet + acl_config: /home/gerrit2/acls/openstack-ci-puppet.config - project: openstack/openstack-manuals - project: openstack/openstack-planet - project: openstack/openstack-puppet diff --git a/modules/pypimirror/files/run_mirror.py b/modules/pypimirror/files/run_mirror.py index ab21588873..b6b07e84c8 100755 --- a/modules/pypimirror/files/run_mirror.py +++ b/modules/pypimirror/files/run_mirror.py @@ -52,7 +52,7 @@ pip_command = '/usr/local/bin/pip install -M -U -I --exists-action=w ' \ run_command(pip_command % "pip") -config = yaml.load(open(PROJECTS_YAML)) +(defaults, config) = [config for config in yaml.load_all(open(PROJECTS_YAML))] for section in config: project = section['project'] diff --git a/modules/pypimirror/manifests/init.pp b/modules/pypimirror/manifests/init.pp index 39e2bd23b2..6255a0e3d9 100644 --- a/modules/pypimirror/manifests/init.pp +++ b/modules/pypimirror/manifests/init.pp @@ -5,6 +5,8 @@ class pypimirror( $pip_download = '/var/lib/pip-download', $pip_cache = '/var/cache/pip', $git_source = 'https://github.com', + $local_git_dir = '/var/lib/git', + $ssh_project_key = 'UNDEF', $projects = [] ) { @@ -47,7 +49,7 @@ class pypimirror( owner => 'root', group => 'root', mode => '0444', - source => 'puppet:///openstack_project/review.projects.yaml', + content => template('openstack_project/review.projects.yaml.erb'), replace => true, }