From 083281f7ee6b17c0a6af39944a5764df5e4d6fc0 Mon Sep 17 00:00:00 2001 From: "James E. Blair" <jeblair@openstack.org> Date: Mon, 17 Feb 2014 08:47:04 -0800 Subject: [PATCH] Add Zuul merge servers Remove obsolete config options from zuul module. The server and merger classes are constructed so that they may coexist on a single server. Also, the init section is constructed so that it will install everything needed for both services but without activating them (leaving maximum flexibility for the operator). Change-Id: I7b86fbbe4611c5edfb463a0a6944e0717f664188 --- manifests/site.pp | 20 +++ .../openstack_project/manifests/zuul_dev.pp | 9 +- .../manifests/zuul_merger.pp | 31 ++++ .../openstack_project/manifests/zuul_prod.pp | 10 +- modules/zuul/files/zuul-merger.init | 134 ++++++++++++++++++ modules/zuul/manifests/init.pp | 37 ++--- modules/zuul/manifests/merger.pp | 36 +++++ modules/zuul/manifests/server.pp | 32 +++++ modules/zuul/templates/zuul.conf.erb | 11 +- 9 files changed, 285 insertions(+), 35 deletions(-) create mode 100644 modules/openstack_project/manifests/zuul_merger.pp create mode 100644 modules/zuul/files/zuul-merger.init create mode 100644 modules/zuul/manifests/merger.pp create mode 100644 modules/zuul/manifests/server.pp diff --git a/manifests/site.pp b/manifests/site.pp index 309a93be48..a5d768f7be 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -518,10 +518,30 @@ node 'zuul.openstack.org' { 'jenkins06.openstack.org', 'jenkins07.openstack.org', 'jenkins-dev.openstack.org', + 'zm01.openstack.org', + 'zm02.openstack.org', ], } } +node 'zm01.openstack.org' { + class { 'openstack_project::zuul_merger': + gerrit_server => 'review.openstack.org', + gerrit_user => 'jenkins', + zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'), + sysadmins => hiera('sysadmins'), + } +} + +node 'zm02.openstack.org' { + class { 'openstack_project::zuul_merger': + gerrit_server => 'review.openstack.org', + gerrit_user => 'jenkins', + zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'), + sysadmins => hiera('sysadmins'), + } +} + node 'zuul-dev.openstack.org' { class { 'openstack_project::zuul_dev': gerrit_server => 'review-dev.openstack.org', diff --git a/modules/openstack_project/manifests/zuul_dev.pp b/modules/openstack_project/manifests/zuul_dev.pp index ce7556fbd2..ca6dcef63d 100644 --- a/modules/openstack_project/manifests/zuul_dev.pp +++ b/modules/openstack_project/manifests/zuul_dev.pp @@ -28,12 +28,14 @@ class openstack_project::zuul_dev( zuul_ssh_private_key => $zuul_ssh_private_key, url_pattern => $url_pattern, zuul_url => $zuul_url, - push_change_refs => false, job_name_in_report => true, status_url => 'http://zuul-dev.openstack.org/', statsd_host => $statsd_host, } + class { '::zuul::server': } + class { '::zuul::merger': } + file { '/etc/zuul/layout.yaml': ensure => present, source => 'puppet:///modules/openstack_project/zuul/layout-dev.yaml', @@ -58,6 +60,11 @@ class openstack_project::zuul_dev( notify => Exec['zuul-reload'], } + file { '/etc/zuul/merger-logging.conf': + ensure => present, + source => 'puppet:///modules/openstack_project/zuul/merger-logging.conf', + } + class { '::recheckwatch': gerrit_server => $gerrit_server, gerrit_user => $gerrit_user, diff --git a/modules/openstack_project/manifests/zuul_merger.pp b/modules/openstack_project/manifests/zuul_merger.pp new file mode 100644 index 0000000000..b6d7eb34f8 --- /dev/null +++ b/modules/openstack_project/manifests/zuul_merger.pp @@ -0,0 +1,31 @@ +# == Class: openstack_project::zuul_merger +# +class openstack_project::zuul_merger( + $vhost_name = $::fqdn, + $gerrit_server = '', + $gerrit_user = '', + $zuul_ssh_private_key = '', + $zuul_url = "http://${::fqdn}/p", + $sysadmins = [], +) { + + class { 'openstack_project::server': + iptables_public_tcp_ports => [80], + sysadmins => $sysadmins, + } + + class { '::zuul': + vhost_name => $vhost_name, + gerrit_server => $gerrit_server, + gerrit_user => $gerrit_user, + zuul_ssh_private_key => $zuul_ssh_private_key, + zuul_url => $zuul_url, + } + + class { '::zuul::merger': } + + file { '/etc/zuul/merger-logging.conf': + ensure => present, + source => 'puppet:///modules/openstack_project/zuul/merger-logging.conf', + } +} diff --git a/modules/openstack_project/manifests/zuul_prod.pp b/modules/openstack_project/manifests/zuul_prod.pp index b5a25bce11..36bb66b7a9 100644 --- a/modules/openstack_project/manifests/zuul_prod.pp +++ b/modules/openstack_project/manifests/zuul_prod.pp @@ -10,7 +10,6 @@ class openstack_project::zuul_prod( $sysadmins = [], $statsd_host = '', $gearman_workers = [], - $replication_targets = [] ) { # Turn a list of hostnames into a list of iptables rules $iptables_rules = regsubst ($gearman_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 4730 -s \1 -j ACCEPT') @@ -29,13 +28,13 @@ class openstack_project::zuul_prod( zuul_ssh_private_key => $zuul_ssh_private_key, url_pattern => $url_pattern, zuul_url => $zuul_url, - push_change_refs => false, job_name_in_report => true, status_url => 'http://status.openstack.org/zuul/', statsd_host => $statsd_host, - replication_targets => $replication_targets, } + class { '::zuul::server': } + file { '/etc/zuul/layout.yaml': ensure => present, source => 'puppet:///modules/openstack_project/zuul/layout.yaml', @@ -60,6 +59,11 @@ class openstack_project::zuul_prod( notify => Exec['zuul-reload'], } + file { '/etc/zuul/merger-logging.conf': + ensure => present, + source => 'puppet:///modules/openstack_project/zuul/merger-logging.conf', + } + class { '::recheckwatch': gerrit_server => $gerrit_server, gerrit_user => $gerrit_user, diff --git a/modules/zuul/files/zuul-merger.init b/modules/zuul/files/zuul-merger.init new file mode 100644 index 0000000000..d0f6742a30 --- /dev/null +++ b/modules/zuul/files/zuul-merger.init @@ -0,0 +1,134 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: zuul +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Zuul +# Description: Trunk gating system +### END INIT INFO + +# Do NOT "set -e" + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Zuul" +NAME=zuul-merger +DAEMON=/usr/local/bin/zuul-merger +PIDFILE=/var/run/$NAME/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +USER=zuul + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + + mkdir -p /var/run/$NAME + chown $USER /var/run/$NAME + ulimit -n 8192 + start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 + # Add code here, if necessary, that waits for the process to be ready + # to handle requests from services started subsequently which depend + # on this one. As a last resort, sleep for some time. +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --signal 9 --pidfile $PIDFILE + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + rm -f /var/run/$NAME/* + return "$RETVAL" +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name zuul-merger + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + log_daemon_msg "Reloading $DESC" "$NAME" + do_reload + log_end_msg $? + ;; + restart) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + do_start + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: diff --git a/modules/zuul/manifests/init.pp b/modules/zuul/manifests/init.pp index c25b05f933..9320a4283c 100644 --- a/modules/zuul/manifests/init.pp +++ b/modules/zuul/manifests/init.pp @@ -29,11 +29,9 @@ class zuul ( $status_url = "https://${::fqdn}/", $zuul_url = '', $git_source_repo = 'https://git.openstack.org/openstack-infra/zuul', - $push_change_refs = false, $job_name_in_report = false, $revision = 'master', $statsd_host = '', - $replication_targets = [] ) { include apache include pip @@ -132,6 +130,13 @@ class zuul ( require => User['zuul'], } + file { '/var/run/zuul-merger': + ensure => directory, + owner => 'zuul', + group => 'zuul', + require => User['zuul'], + } + file { '/var/lib/zuul': ensure => directory, owner => 'zuul', @@ -210,27 +215,12 @@ class zuul ( source => 'puppet:///modules/zuul/zuul.init', } - exec { 'zuul-reload': - command => '/etc/init.d/zuul reload', - require => File['/etc/init.d/zuul'], - refreshonly => true, - } - - service { 'zuul': - name => 'zuul', - enable => true, - hasrestart => true, - require => File['/etc/init.d/zuul'], - } - - cron { 'zuul_repack': - user => 'zuul', - hour => '4', - minute => '7', - command => 'find /var/lib/zuul/git/ -maxdepth 3 -type d -name ".git" -exec git --git-dir="{}" pack-refs --all \;', - environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin', - require => [User['zuul'], - File['/var/lib/zuul/git']], + file { '/etc/init.d/zuul-merger': + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => 'puppet:///modules/zuul/zuul-merger.init', } apache::vhost { $vhost_name: @@ -248,5 +238,4 @@ class zuul ( a2mod { 'proxy_http': ensure => present, } - } diff --git a/modules/zuul/manifests/merger.pp b/modules/zuul/manifests/merger.pp new file mode 100644 index 0000000000..abda6acb34 --- /dev/null +++ b/modules/zuul/manifests/merger.pp @@ -0,0 +1,36 @@ +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# Copyright 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# == Class: zuul::merger +# +class zuul::merger ( +) { + service { 'zuul-merger': + name => 'zuul-merger', + enable => true, + hasrestart => true, + require => File['/etc/init.d/zuul-merger'], + } + + cron { 'zuul_repack': + user => 'zuul', + hour => '4', + minute => '7', + command => 'find /var/lib/zuul/git/ -maxdepth 3 -type d -name ".git" -exec git --git-dir="{}" pack-refs --all \;', + environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin', + require => [User['zuul'], + File['/var/lib/zuul/git']], + } +} diff --git a/modules/zuul/manifests/server.pp b/modules/zuul/manifests/server.pp new file mode 100644 index 0000000000..4e06c3054e --- /dev/null +++ b/modules/zuul/manifests/server.pp @@ -0,0 +1,32 @@ +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# Copyright 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# == Class: zuul::server +# +class zuul::server ( +) { + service { 'zuul': + name => 'zuul', + enable => true, + hasrestart => true, + require => File['/etc/init.d/zuul'], + } + + exec { 'zuul-reload': + command => '/etc/init.d/zuul reload', + require => File['/etc/init.d/zuul'], + refreshonly => true, + } +} diff --git a/modules/zuul/templates/zuul.conf.erb b/modules/zuul/templates/zuul.conf.erb index cbf08359b3..e38198d9d8 100644 --- a/modules/zuul/templates/zuul.conf.erb +++ b/modules/zuul/templates/zuul.conf.erb @@ -15,15 +15,12 @@ layout_config=/etc/zuul/layout.yaml log_config=/etc/zuul/logging.conf state_dir=/var/lib/zuul git_dir=/var/lib/zuul/git -push_change_refs=<%= push_change_refs %> url_pattern=<%= url_pattern %> status_url=<%= status_url %> job_name_in_report=<%= job_name_in_report %> zuul_url=<%= zuul_url %> -<% if replication_targets != [] -%> -[replication] -<% replication_targets.each do |target| -%> -<%= target['name'] %>=<%= target['url'] %> -<% end -%> -<% end -%> +[merger] +git_dir=/var/lib/zuul/git +zuul_url=<%= zuul_url %> +log_config=/etc/zuul/merger-logging.conf