Groups portal infra code refactor

This patch removes the drush make site building function from
groups-dev instance, and now directly fetch release tarballs from
http://tarballs.openstack.org/groups repository. With an advanced
multi-slot deployment architecture it prevents the typical Drupal
WSOD issues that randomly caused site malfunction when a request
arrived during installation. It also simplifies the
deployment steps using the standard drush aliases and
drush-dsd extension and supports local configuration variables
in local_settings.php file.

Change-Id: I73976a60e080d15b6f513db79fee46bcf468e302
This commit is contained in:
Marton Kiss 2014-06-11 22:53:47 +02:00
parent 1b88b6ccdd
commit 9adc5fd79b
8 changed files with 254 additions and 428 deletions

View File

@ -1,223 +0,0 @@
#!/bin/bash
#
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Drupal site deploy tool. Install and setup Drupal from a distribution file.
#
# See drupal_site_deploy.sh --help for further parameters and examples.
#
# Basic deployment flow:
# 1. clean-up destination directory (optional)
# 2. extract dist tarball to destination with proper permissions,
# create settings.php and files directory under sites/default
# 3. install drupal with drush si command, setup admin password and
# proper filename, and repair sites/default/files ownership
# 4. create flag-file that marks successfull installation (optional)
site_admin_user=admin
profile_name=standard
file_group=www-data
file_owner=ubuntu
_print_help() {
echo "$(basename "$0") -- Deploy and configure a Drupal site from distribution file
## Global options ##
-st, --site-tarball <tar-file> source tarball file used for deployment
-db, --db-url <db-url> database url: mysql://user:pass@hostname[:port]/dbname
-sn, --site-name <sitename> name of the website
-su, --site-admin-user <username> username of admin account
-sp, --site-admin-password <password> password of admin account
-pn, --profile-name <profilename> profile used to install (defaults to standard)
-p, --prepare prepare for deployment, but skip the install phase
-c, --clean clean target directory
-dst, --dest-dir <target-directory> target directory
-ff, --flag-file <flagfile> create a flagfile after successfull run
-bu, --base-url <base-url> base_url parameter of settings.php
-in, --config read parameters from configuration file
-fo, --file-owner <user> file owner
-fg, --file-group <group> group owner
-h, --help display this help
## Examples ##
install using cli parameters:
drupal_site_deploy.sh -st drupal-7.23.tar.gz -pn standard \\
--db-url mysql://root:pass@localhost:port/dbname \\
--site-name drupal-dev.local \\
--site-admin-user admin \\
--site-admin-password Wr5pUF@f8*Wr
install using config file params:
drupal_site_deploy.sh -in l10n-dev.config
"
}
_set_args() {
while [ "$1" != "" ]; do
case $1 in
"-st" | "--site-tarball")
shift
site_tarball=$1
;;
"-db" | "--db-url")
shift
db_url=$1
;;
"-sn" | "--site-name")
shift
site_name=$1
;;
"-su" | "--site-admin-user")
shift
site_admin_user=$1
;;
"-sp" | "--site-admin-password")
shift
site_admin_password=$1
;;
"-pn" | "--profile-name")
shift
profile_name=$1
;;
"-p" | "--prepare")
is_prepare=true
;;
"-c" | "--clean")
is_clean=true
;;
"-dst" | "--dest-dir")
shift
dest_dir=$1
;;
"-ff" | "--flag-file")
shift
flag_file=$1
;;
"-bu" | "--base-url")
shift
base_url=$1
;;
"-in" | "--config-file")
shift
config_rc=$1
;;
"-fo" | "--file-owner")
shift
file_owner=$1
;;
"-fg" | "--file-group")
shift
file_group=$1
;;
"-h" | "--help")
_print_help
exit 1
;;
esac
shift
done
}
_validate_args() {
if [ ! $dest_dir ]; then
echo "Error: Mandatory parameter destination directory is missing."
exit 1
fi
if [ ! -d $dest_dir ]; then
echo "Error: Invalid destination directory: $dest_dir"
exit 1
fi
if [ ! $site_tarball ]; then
echo "Error: Mandatory parameter site tarball is missing."
exit 1
fi
if [ ! -f $site_tarball ]; then
echo "Error: Invalid site tarball file: $site_tarball"
exit 1
fi
if [ ! $db_url ]; then
echo "Error: Mandatory parameter db url is missing."
exit 1
fi
}
_stage_cleanup() {
echo "cleanup site directory"
rm -rf $dest_dir/*
}
_stage_prepare() {
echo "prepare installation"
umask 0027
tar xfz $site_tarball --strip-components=1 --no-same-permissions -C $dest_dir
cp $dest_dir/sites/default/default.settings.php $dest_dir/sites/default/settings.php
if [ $base_url ]; then
echo "\$base_url='$base_url';" >> $dest_dir/sites/default/settings.php
fi
mkdir -p $dest_dir/sites/default/files
chmod g+rwxs $dest_dir/sites/default/files
chown -R $file_owner:$file_group $dest_dir
chmod 0664 $dest_dir/sites/default/settings.php
umask 0022
}
_stage_install() {
echo "install distribution file"
save_dir=`pwd`
cd $dest_dir
drush si -y $profile_name --db-url=$db_url
chmod 0440 $dest_dir/sites/default/settings.php
chgrp -R www-data $dest_dir/sites/default/files
if [ $site_name ]; then
drush vset site_name $site_name
fi
if [ $site_admin_password ]; then
drush upwd $site_admin_user --password="$site_admin_password"
fi
drush features-revert-all -y
cd $save_dir
}
set -e
_set_args $*
if [ -f $config_rc ]; then
source $config_rc
fi
_validate_args
if [ $is_clean ]; then
_stage_cleanup
fi
_stage_prepare
if [ ! $is_prepare ]; then
_stage_install
fi
if [ $flag_file ]; then
touch $flag_file
chown root:root $flag_file
fi
echo "sitedeploy completed in $SECONDS seconds."

View File

@ -1,68 +0,0 @@
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# == Define: distbuild
#
# define to build distribution from git makefile
#
define drupal::distbuild (
$site_sandbox_root = undef,
$site_staging_root = undef,
$site_repo_url = undef,
$site_repo_branch = 'master',
$site_build_repo_name = undef,
$site_staging_tarball = undef,
$site_build_flagfile = undef,
$site_deploy_flagfile = undef,
$site_makefile = undef,
) {
file { $site_sandbox_root:
ensure => directory,
owner => 'root',
group => 'root',
mode => '0755',
}
file { $site_staging_root:
ensure => directory,
owner => 'root',
group => 'root',
mode => '0755',
}
vcsrepo { "${site_sandbox_root}/${site_build_repo_name}":
ensure => latest,
provider => git,
revision => $site_repo_branch,
source => $site_repo_url,
}
exec { 'drupal-build-dist':
path => '/usr/bin:/bin',
timeout => 900,
cwd => "${site_sandbox_root}/${$site_build_repo_name}",
command => "rm -rf ${site_staging_root}/${site_staging_tarball} && drush make --tar ${site_makefile} ${site_staging_root}/${site_staging_tarball}",
unless => "diff ${site_sandbox_root}/${$site_build_repo_name}/.git/refs/heads/master ${site_build_flagfile}",
require => File[$site_staging_root],
subscribe => Vcsrepo["${site_sandbox_root}/${$site_build_repo_name}"],
}
exec { 'drupal-build-dist-post':
path => '/usr/bin:/bin',
command => "cp ${site_sandbox_root}/${$site_build_repo_name}/.git/refs/heads/master ${site_build_flagfile} && rm -rf ${site_deploy_flagfile}",
unless => "diff ${site_sandbox_root}/${$site_build_repo_name}/.git/refs/heads/master ${site_build_flagfile}",
subscribe => Vcsrepo["${site_sandbox_root}/${$site_build_repo_name}"],
require => Exec['drupal-build-dist'],
}
}

75
manifests/drush.pp Normal file
View File

@ -0,0 +1,75 @@
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# == Define: drush
#
# define to add drush and custom dsd extension
#
# Drush parameters:
# - drushdsdtar: drush dsd release tarball
# - basedrushdsdtar: drush dsd tar local filename
# - download_dir: download directory, local copy of release tarball lives here
define drupal::drush (
$drushdsdtar = 'https://github.com/mkissam/drush-dsd/archive/v0.7.tar.gz',
$basedrushdsdtar = 'drush-dsd-0.7.tar.gz',
$download_dir = '/srv/downloads',
) {
# pear / drush cli tool
pear::package { 'PEAR': }
pear::package { 'Console_Table': }
pear::package { 'drush':
version => '6.0.0',
repository => 'pear.drush.org',
require => [ Pear::Package['PEAR'], Pear::Package['Console_Table'] ],
}
file { '/usr/share/php/drush/commands/dsd':
ensure => directory,
owner => 'root',
group => 'root',
mode => '0755',
require => [ Pear::Package['drush'] ]
}
# If we don't already have the specified dsd tar, download it.
exec { "download:${drushdsdtar}":
command => "/usr/bin/wget ${drushdsdtar} -O ${download_dir}/${basedrushdsdtar}",
creates => "${download_dir}/${basedrushdsdtar}",
require => File[$download_dir],
}
# If drush-dsd.tar.gz isn't the same as $basedrushdsdtar, install it.
file { "${download_dir}/drush-dsd.tar.gz":
ensure => present,
source => "file://${download_dir}/${basedrushdsdtar}",
require => Exec["download:${drushdsdtar}"],
replace => true,
owner => 'root',
group => 'root',
mode => '0644',
}
# If drush-dsd just created extract to /etc/drush
exec { 'drush-dsd-initial-init':
user => 'root',
command => "/bin/tar -C /usr/share/php/drush/commands/dsd --strip 1 -xzvf ${download_dir}/drush-dsd.tar.gz;/usr/bin/drush cc all",
subscribe => File["${download_dir}/drush-dsd.tar.gz"],
refreshonly => true,
logoutput => true,
require => File['/usr/share/php/drush/commands/dsd'],
}
}

View File

@ -19,8 +19,10 @@
#
# Actions:
# - Prepare apache vhost and create mysql database (optional)
# - Build distribution tarball from git repo as a soruce
# - Deploy dist tarball and setup Drupal from scratch
# - Install Drush tool with Drush-dsd extension
# - Fetch distribution tarball from remote repository
# - Deploy dist tarball and setup Drupal from scratch or
# upgrade an existing site.
#
# Site parameters:
# - site_name: name of the site (FQDN for example)
@ -28,6 +30,8 @@
# - site_docroot: root directory of drupal site
# - site_vhost_root: root directory of virtual hosts
# - site_create_database: if true, create a new database (default: false)
# - site_alias: drush site alias name
# - site_profile: installation profile to deploy
#
# Mysql connection:
# - mysql_user: mysql user of drupal site
@ -35,39 +39,37 @@
# - mysql_database: site database name
# - mysql_host: host of mysql server (default: localhost)
#
# Distribution build process:
# - site_sandbox_root: root directory of sandbox where build happens
# - site_staging_root: root directory of target tarballs
# - site_staging_tarball: target tarball of build process
# - site_makefile: installation profile drush makefile
# - site_build_reponame: local repository name under sandbox root
# - site_repo_url: git repo url of installation profile source
# - site_build_flagfile: triggers a rebuild when missing or git head differs
# Drupal configuration variables:
# - conf_cron_key: cron_key setting used for cron access
#
# Remarks:
# - the site lives in /srv/vhosts/{hostname}/slot0 or slot1 directory
# - the /srv/vhosts/{hostname}/w symlinks to slot0 or slot1 and
# points to actual site root. The upgrade process will begin in
# inactive slot so we can avoid typical WSOD issues with Drupal.
# - for temporary package/tarball download it is using the
# /srv/downloads directory.
#
# Deploy process:
# - site_profile: installation profile to deploy
# - site_deploy_flagfile: triggers a redeploy when this flagfile is missing
class drupal (
$site_name = undef,
$site_docroot = undef,
$site_root = undef,
$site_docroot = "${site_root}/w",
$site_mysql_host = 'localhost',
$site_mysql_user = undef,
$site_mysql_password = undef,
$site_mysql_database = undef,
$site_vhost_root = '/srv/vhosts',
$site_sandbox_root = '/srv/sandbox',
$site_staging_root = '/srv/sandbox/release',
$site_staging_tarball = '',
$site_profile = 'standard',
$site_admin_password = undef,
$site_build_reponame = undef,
$site_makefile = undef,
$site_repo_url = undef,
$site_build_flagfile = '/tmp/drupal-site-build',
$site_deploy_flagfile = '/tmp/drupal-site-deploy',
$site_alias = undef,
$site_create_database = false,
$site_base_url = false,
$site_file_owner = 'root',
$package_repository = undef,
$package_branch = undef,
$conf_cron_key = undef,
$conf_markdown_directory = undef,
) {
include apache
include pear
@ -84,11 +86,11 @@ class drupal (
port => 80,
priority => '50',
docroot => $site_docroot,
require => File[$site_docroot],
require => Exec['init-slot-dirs'],
template => 'drupal/drupal.vhost.erb',
}
file { $site_docroot:
file { $site_root:
ensure => directory,
owner => 'root',
group => 'www-data',
@ -96,6 +98,16 @@ class drupal (
require => Package['httpd'],
}
# Create initial symlink here to allow apache vhost creation
# so drush dsd can flip this symlink between slot0/slot1
# (won't be recreated until the symlink exists)
exec { 'init-slot-dirs':
command => "/bin/ln -s ${site_root}/slot1 ${site_docroot}",
unless => "/usr/bin/test -L ${site_docroot}",
logoutput => 'on_failure',
require => File[$site_root],
}
a2mod { 'rewrite':
ensure => present,
}
@ -110,13 +122,18 @@ class drupal (
notify => Service['httpd'],
}
# pear / drush cli tool
pear::package { 'PEAR': }
pear::package { 'Console_Table': }
pear::package { 'drush':
version => '6.0.0',
repository => 'pear.drush.org',
require => [ Pear::Package['PEAR'], Pear::Package['Console_Table'] ],
# This directory is used to download and cache tarball releases
# without proper upstream packages
file { '/srv/downloads':
ensure => directory,
owner => 'root',
group => 'root',
mode => '0755',
}
# setup drush and drush-dsd extension
drush { 'drush':
require => File['/srv/downloads'],
}
# site mysql database
@ -130,34 +147,95 @@ class drupal (
}
}
# drupal dist-build
distbuild { "distbuild-${site_name}":
site_sandbox_root => $site_sandbox_root,
site_staging_root => $site_staging_root,
site_repo_url => $site_repo_url,
site_build_repo_name => $site_build_reponame,
site_staging_tarball => $site_staging_tarball,
site_build_flagfile => $site_build_flagfile,
site_deploy_flagfile => $site_deploy_flagfile,
site_makefile => $site_makefile,
require => [ Package['httpd'], Pear::Package['drush'] ],
# drush site-alias definition
file { '/etc/drush':
ensure => directory,
owner => 'root',
group => 'root',
mode => '0755',
}
# drupal site deploy
sitedeploy { "sitedeploy-${site_name}":
site_docroot => $site_docroot,
site_staging_root => $site_staging_root,
site_staging_tarball => $site_staging_tarball,
site_deploy_flagfile => $site_deploy_flagfile,
site_name => $site_name,
site_profile => $site_profile,
site_mysql_user => $site_mysql_user,
site_mysql_password => $site_mysql_password,
site_mysql_host => $site_mysql_host,
site_mysql_database => $site_mysql_database,
site_admin_password => $site_admin_password,
site_base_url => $site_base_url,
require => Distbuild["distbuild-${site_name}"],
file { '/etc/drush/aliases.drushrc.php':
ensure => present,
owner => 'root',
group => 'root',
mode => '0400',
content => template('drupal/aliases.drushrc.php.erb'),
replace => true,
require => [ File['/etc/drush'], Drush['drush'] ],
}
}
# site custom configuration
file { "${site_root}/etc":
ensure => directory,
owner => 'root',
group => 'root',
mode => '0755',
require => File[$site_root],
}
file { "${site_root}/etc/settings.php":
ensure => file,
owner => 'root',
group => 'root',
mode => '0400',
content => template('drupal/settings.php.erb'),
replace => true,
require => File["${site_root}/etc"],
}
# add site distro tarball from http repository including
# md5 hash file
exec { "download:${package_branch}.tar.gz":
command => "/usr/bin/wget ${package_repository}/${package_branch}.tar.gz -O /srv/downloads/${package_branch}.tar.gz",
creates => "/srv/downloads/${package_branch}.tar.gz",
logoutput => 'on_failure',
require => File['/srv/downloads'],
}
exec { "download:${package_branch}.md5":
command => "/usr/bin/wget ${package_repository}/${package_branch}.md5 -O /srv/downloads/${package_branch}.md5",
creates => "/srv/downloads/${package_branch}.md5",
logoutput => 'on_failure',
require => [ Exec["download:${package_branch}.tar.gz"] ],
}
# deploy a site from scratch when site status is 'NOT INSTALLED'
exec { "sitedeploy-${site_name}":
command => "/usr/bin/drush dsd-init @${site_alias} /srv/downloads/${package_branch}.tar.gz",
logoutput => true,
timeout => 600,
onlyif => "/usr/bin/drush dsd-status @${site_alias} | /bin/grep -c 'NOT INSTALLED'",
require => [
Exec["download:${package_branch}.md5"],
File['/etc/drush/aliases.drushrc.php'],
]
}
# update the site into a new slot when a remote update available
exec { "siteupdate-${site_name}":
command => "/usr/bin/drush dsd-update @${site_alias} /srv/downloads/${package_branch}.tar.gz",
logoutput => true,
timeout => 600,
onlyif => "/usr/bin/drush dsd-status @${site_alias} | /bin/grep -c 'UPDATE'",
require => [
Exec["download:${package_branch}.md5"],
File['/etc/drush/aliases.drushrc.php'],
]
}
# setup cron job
cron { $site_name:
name => "${site_name}.cron",
command => "wget -O - -q -t 1 ${$site_base_url}/cron.php?cron_key=${$conf_cron_key}",
user => root,
minute => '*/5',
require => [
Exec["sitedeploy-${site_name}"],
]
}
}

View File

@ -1,66 +0,0 @@
# Copyright 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# == Define: sitedeploy
#
# define to deploy drupal site from distribution tarball
#
define drupal::sitedeploy (
$site_docroot = undef,
$site_staging_root = undef,
$site_staging_tarball = undef,
$site_deploy_flagfile = undef,
$site_name = undef,
$site_profile = undef,
$site_mysql_user = undef,
$site_mysql_password = undef,
$site_mysql_host = undef,
$site_mysql_database = undef,
$site_admin_password = '',
$site_deploy_timeout = 600,
$site_base_url = undef,
$site_file_owner = 'root',
) {
file { '/usr/local/sbin/drupal-site-deploy.sh':
ensure => present,
owner => 'root',
group => 'root',
mode => '0744',
source => 'puppet:///modules/drupal/drupal_site_deploy.sh',
}
file { '/etc/drupal_site':
ensure => directory,
}
file { "/etc/drupal_site/${site_name}.config":
ensure => present,
owner => 'root',
group => 'root',
mode => '0400',
content => template('drupal/site.config.erb'),
replace => true,
require => File['/etc/drupal_site'],
}
exec { "drupal-deploy-${site_name}":
path => '/usr/bin:/bin:/usr/local/sbin',
command => "drupal-site-deploy.sh -in /etc/drupal_site/${site_name}.config",
creates => $site_deploy_flagfile,
timeout => $site_deploy_timeout,
require => [ File["/etc/drupal_site/${site_name}.config"],
File['/usr/local/sbin/drupal-site-deploy.sh'] ],
}
}

View File

@ -0,0 +1,29 @@
<?php
$aliases['<%= @site_alias %>'] = array(
'root' => '<%= @site_docroot %>',
'dsd-root' => '<%= @site_root %>',
'uri' => '<%= @site_base_url %>',
'db-url' => 'mysql://<%= @site_mysql_user %>:<%= @site_mysql_password %>@<%= @site_mysql_host %>/<%= @site_mysql_database %>',
'databases' => array(
'default' => array(
'driver' => 'mysql',
'username' => '<%= @site_mysql_user %>',
'password' => '<%= @site_mysql_password %>',
'port' => '',
'host' => '<%= @site_mysql_host %>',
'database' => '<%= @site_mysql_database %>',
),
),
'file-owner' => '<%= @site_file_owner %>',
'file-group' => 'www-data',
'variables' => array(
'site_name' => '<%= @site_name %>',
),
'profile' => '<%= @site_profile %>',
'default-admin-password' => '<%= @site_admin_password %>',
'disable-features-revert' => FALSE,
'package-provider' => 'static-tarball',
'package-repository' => '<%= @package_repository %>',
'package-branch' => '<%= @package_branch %>',
);

View File

@ -0,0 +1,14 @@
<?php
/**
* @file
* Drupal configuration overrides
*
*/
<% if @conf_cron_key %>
$conf['cron_key'] = '<%= @conf_cron_key %>';
<% end %>
<% if @conf_markdown_directory %>
$conf['groups_feeds_markdown_directory'] = '<%= @conf_markdown_directory %>';
<% end %>

View File

@ -1,13 +0,0 @@
# Site deployment config file: <%= @site_name %>
#
site_tarball=<%= @site_staging_root %>/<%= @site_staging_tarball %>
db_url=mysql://<%= @site_mysql_user %>:<%= @site_mysql_password %>@<%= @site_mysql_host %>/<%= @site_mysql_database %>
site_admin_password=<%= @site_admin_password %>
site_name=<%= @site_name %>
profile_name=<%= @site_profile %>
dest_dir=<%= @site_docroot %>
flag_file=<%= @site_deploy_flagfile %>
is_clean=true
base_url=<%= @site_base_url %>
file_owner=<%= @site_file_owner %>