From 7fa0616201e13e536bbc9e6fbd125c67d9ea90a5 Mon Sep 17 00:00:00 2001 From: K Jonathan Harker Date: Tue, 22 Apr 2014 15:16:18 -0700 Subject: [PATCH] Move openstack-specific config out of ::jenkins The ::jenkins::slave class contained a lot of openstack-specific configuration rather than configuration of a generic jenkins slave. The term "bare slave" is overloaded and confusing: create simple_slave and thick_slave to differentiate between the two meanings of "bare". Some portions of ::jenkins::slave will move to simple_slave, some portions to thick_slave, and some portions to slave_common (all in the openstack_project module). Change-Id: I5281a03a7f6da3f98714bcc59ae840ace8435578 --- .../files/nodepool/scripts/prepare_node.sh | 6 +- .../nodepool/scripts/prepare_node_bare.sh | 4 +- .../nodepool/scripts/prepare_node_devstack.sh | 4 +- .../prepare_node_devstack_new_kernel.sh | 4 +- .../nodepool/scripts/prepare_node_py3k.sh | 4 +- .../nodepool/scripts/prepare_node_tripleo.sh | 4 +- .../slave_scripts/baremetal-archive-logs.sh | 5 + .../files/slave_scripts/baremetal-deploy.sh | 44 ++ .../slave_scripts/baremetal-os-install.sh | 22 + .../files/slave_scripts/bump-milestone.sh | 56 ++ .../files/slave_scripts/create-ppa-package.sh | 78 ++ .../files/slave_scripts/docbook-properties.sh | 28 + .../files/slave_scripts/functions.sh | 49 ++ .../files/slave_scripts/gerrit-git-prep.sh | 97 +++ .../files/slave_scripts/jenkins-oom-grep.sh | 39 + .../files/slave_scripts/jenkins-sudo-grep.sh | 62 ++ .../files/slave_scripts/jenkinsci-upload.sh | 46 ++ .../files/slave_scripts/lvm-kexec-reset.sh | 7 + .../files/slave_scripts/markdown-docbook.sh | 21 + .../slave_scripts/mavencentral-upload.sh | 44 ++ .../files/slave_scripts/merge_tags.sh | 30 + .../files/slave_scripts/package-gerrit.sh | 7 + .../files/slave_scripts/php-laravel-build.sh | 12 + .../files/slave_scripts/ping.py | 11 + .../project-requirements-change.py | 155 ++++ .../propose_requirements_update.sh | 97 +++ .../propose_translation_update.sh | 84 ++ .../propose_translation_update_horizon.sh | 92 +++ .../propose_translation_update_manuals.sh | 128 +++ .../files/slave_scripts/pypi-extract-name.py | 23 + .../files/slave_scripts/pypi-upload.sh | 36 + .../files/slave_scripts/run-bash8.sh | 27 + .../files/slave_scripts/run-cover.sh | 32 + .../files/slave_scripts/run-docs.sh | 84 ++ .../files/slave_scripts/run-jsbuild.sh | 40 + .../files/slave_scripts/run-pep8.sh | 28 + .../files/slave_scripts/run-pylint.sh | 28 + .../files/slave_scripts/run-selenium.sh | 37 + .../files/slave_scripts/run-tarball.sh | 38 + .../files/slave_scripts/run-tox.sh | 62 ++ .../files/slave_scripts/run-unittests.sh | 118 +++ .../files/slave_scripts/run-xmllint.sh | 4 + .../files/slave_scripts/select-mirror.sh | 57 ++ .../files/slave_scripts/subunit2html.py | 727 ++++++++++++++++++ .../files/slave_scripts/tardiff.py | 189 +++++ .../files/slave_scripts/update-pip-cache.sh | 10 + .../upstream_translation_horizon.sh | 59 ++ .../upstream_translation_update.sh | 46 ++ .../upstream_translation_update_manuals.sh | 70 ++ .../files/slave_scripts/version-properties.sh | 29 + .../files/slave_scripts/wait_for_nova.sh | 12 + .../files/slave_scripts/wait_for_puppet.sh | 21 + .../files/slave_scripts/zuul_swift_upload.py | 142 ++++ 53 files changed, 3146 insertions(+), 13 deletions(-) create mode 100755 modules/openstack_project/files/slave_scripts/baremetal-archive-logs.sh create mode 100755 modules/openstack_project/files/slave_scripts/baremetal-deploy.sh create mode 100755 modules/openstack_project/files/slave_scripts/baremetal-os-install.sh create mode 100644 modules/openstack_project/files/slave_scripts/bump-milestone.sh create mode 100755 modules/openstack_project/files/slave_scripts/create-ppa-package.sh create mode 100755 modules/openstack_project/files/slave_scripts/docbook-properties.sh create mode 100755 modules/openstack_project/files/slave_scripts/functions.sh create mode 100755 modules/openstack_project/files/slave_scripts/gerrit-git-prep.sh create mode 100755 modules/openstack_project/files/slave_scripts/jenkins-oom-grep.sh create mode 100755 modules/openstack_project/files/slave_scripts/jenkins-sudo-grep.sh create mode 100644 modules/openstack_project/files/slave_scripts/jenkinsci-upload.sh create mode 100755 modules/openstack_project/files/slave_scripts/lvm-kexec-reset.sh create mode 100755 modules/openstack_project/files/slave_scripts/markdown-docbook.sh create mode 100644 modules/openstack_project/files/slave_scripts/mavencentral-upload.sh create mode 100755 modules/openstack_project/files/slave_scripts/merge_tags.sh create mode 100644 modules/openstack_project/files/slave_scripts/package-gerrit.sh create mode 100644 modules/openstack_project/files/slave_scripts/php-laravel-build.sh create mode 100755 modules/openstack_project/files/slave_scripts/ping.py create mode 100755 modules/openstack_project/files/slave_scripts/project-requirements-change.py create mode 100755 modules/openstack_project/files/slave_scripts/propose_requirements_update.sh create mode 100755 modules/openstack_project/files/slave_scripts/propose_translation_update.sh create mode 100755 modules/openstack_project/files/slave_scripts/propose_translation_update_horizon.sh create mode 100755 modules/openstack_project/files/slave_scripts/propose_translation_update_manuals.sh create mode 100644 modules/openstack_project/files/slave_scripts/pypi-extract-name.py create mode 100755 modules/openstack_project/files/slave_scripts/pypi-upload.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-bash8.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-cover.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-docs.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-jsbuild.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-pep8.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-pylint.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-selenium.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-tarball.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-tox.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-unittests.sh create mode 100755 modules/openstack_project/files/slave_scripts/run-xmllint.sh create mode 100755 modules/openstack_project/files/slave_scripts/select-mirror.sh create mode 100755 modules/openstack_project/files/slave_scripts/subunit2html.py create mode 100755 modules/openstack_project/files/slave_scripts/tardiff.py create mode 100755 modules/openstack_project/files/slave_scripts/update-pip-cache.sh create mode 100755 modules/openstack_project/files/slave_scripts/upstream_translation_horizon.sh create mode 100755 modules/openstack_project/files/slave_scripts/upstream_translation_update.sh create mode 100755 modules/openstack_project/files/slave_scripts/upstream_translation_update_manuals.sh create mode 100755 modules/openstack_project/files/slave_scripts/version-properties.sh create mode 100755 modules/openstack_project/files/slave_scripts/wait_for_nova.sh create mode 100755 modules/openstack_project/files/slave_scripts/wait_for_puppet.sh create mode 100755 modules/openstack_project/files/slave_scripts/zuul_swift_upload.py diff --git a/modules/openstack_project/files/nodepool/scripts/prepare_node.sh b/modules/openstack_project/files/nodepool/scripts/prepare_node.sh index fdbf096b06..2f9cab4e4a 100755 --- a/modules/openstack_project/files/nodepool/scripts/prepare_node.sh +++ b/modules/openstack_project/files/nodepool/scripts/prepare_node.sh @@ -18,7 +18,7 @@ HOSTNAME=$1 SUDO=$2 -BARE=$3 +THIN=$3 PYTHON3=${4:-false} PYPY=${5:-false} ALL_MYSQL_PRIVS=${6:-false} @@ -42,10 +42,10 @@ sudo git clone --depth=1 git://git.openstack.org/openstack-infra/config.git \ sudo /bin/bash /root/config/install_modules.sh if [ -z "$NODEPOOL_SSH_KEY" ] ; then sudo puppet apply --modulepath=/root/config/modules:/etc/puppet/modules \ - -e "class {'openstack_project::single_use_slave': sudo => $SUDO, bare => $BARE, python3 => $PYTHON3, include_pypy => $PYPY, all_mysql_privs => $ALL_MYSQL_PRIVS, }" + -e "class {'openstack_project::single_use_slave': sudo => $SUDO, thin => $THIN, python3 => $PYTHON3, include_pypy => $PYPY, all_mysql_privs => $ALL_MYSQL_PRIVS, }" else sudo puppet apply --modulepath=/root/config/modules:/etc/puppet/modules \ - -e "class {'openstack_project::single_use_slave': install_users => false, sudo => $SUDO, bare => $BARE, python3 => $PYTHON3, include_pypy => $PYPY, all_mysql_privs => $ALL_MYSQL_PRIVS, ssh_key => '$NODEPOOL_SSH_KEY', }" + -e "class {'openstack_project::single_use_slave': install_users => false, sudo => $SUDO, thin => $THIN, python3 => $PYTHON3, include_pypy => $PYPY, all_mysql_privs => $ALL_MYSQL_PRIVS, ssh_key => '$NODEPOOL_SSH_KEY', }" fi # The puppet modules should install unbound. Take the nameservers diff --git a/modules/openstack_project/files/nodepool/scripts/prepare_node_bare.sh b/modules/openstack_project/files/nodepool/scripts/prepare_node_bare.sh index 31597067e4..c41ae01772 100755 --- a/modules/openstack_project/files/nodepool/scripts/prepare_node_bare.sh +++ b/modules/openstack_project/files/nodepool/scripts/prepare_node_bare.sh @@ -18,11 +18,11 @@ HOSTNAME=$1 SUDO='true' -BARE='false' +THIN='false' PYTHON3='false' PYPY='false' ALL_MYSQL_PRIVS='true' -./prepare_node.sh "$HOSTNAME" "$SUDO" "$BARE" "$PYTHON3" "$PYPY" "$ALL_MYSQL_PRIVS" +./prepare_node.sh "$HOSTNAME" "$SUDO" "$THIN" "$PYTHON3" "$PYPY" "$ALL_MYSQL_PRIVS" ./restrict_memory.sh diff --git a/modules/openstack_project/files/nodepool/scripts/prepare_node_devstack.sh b/modules/openstack_project/files/nodepool/scripts/prepare_node_devstack.sh index 3b890686c8..45c1e885fd 100755 --- a/modules/openstack_project/files/nodepool/scripts/prepare_node_devstack.sh +++ b/modules/openstack_project/files/nodepool/scripts/prepare_node_devstack.sh @@ -18,9 +18,9 @@ HOSTNAME=$1 SUDO='true' -BARE='true' +THIN='true' -./prepare_node.sh "$HOSTNAME" "$SUDO" "$BARE" +./prepare_node.sh "$HOSTNAME" "$SUDO" "$THIN" sudo -u jenkins -i /opt/nodepool-scripts/prepare_devstack.sh $HOSTNAME ./restrict_memory.sh diff --git a/modules/openstack_project/files/nodepool/scripts/prepare_node_devstack_new_kernel.sh b/modules/openstack_project/files/nodepool/scripts/prepare_node_devstack_new_kernel.sh index 0b6a485752..c2496ab09c 100755 --- a/modules/openstack_project/files/nodepool/scripts/prepare_node_devstack_new_kernel.sh +++ b/modules/openstack_project/files/nodepool/scripts/prepare_node_devstack_new_kernel.sh @@ -18,9 +18,9 @@ HOSTNAME=$1 SUDO='true' -BARE='true' +THIN='true' -./prepare_node.sh "$HOSTNAME" "$SUDO" "$BARE" +./prepare_node.sh "$HOSTNAME" "$SUDO" "$THIN" sudo -u jenkins -i /opt/nodepool-scripts/prepare_devstack_new_kernel.sh $HOSTNAME ./restrict_memory.sh diff --git a/modules/openstack_project/files/nodepool/scripts/prepare_node_py3k.sh b/modules/openstack_project/files/nodepool/scripts/prepare_node_py3k.sh index c970be81b8..9b8be24cbb 100755 --- a/modules/openstack_project/files/nodepool/scripts/prepare_node_py3k.sh +++ b/modules/openstack_project/files/nodepool/scripts/prepare_node_py3k.sh @@ -18,11 +18,11 @@ HOSTNAME=$1 SUDO='false' -BARE='false' +THIN='false' PYTHON3='true' PYPY='true' ALL_MYSQL_PRIVS='true' -./prepare_node.sh "$HOSTNAME" "$SUDO" "$BARE" "$PYTHON3" "$PYPY" "$ALL_MYSQL_PRIVS" +./prepare_node.sh "$HOSTNAME" "$SUDO" "$THIN" "$PYTHON3" "$PYPY" "$ALL_MYSQL_PRIVS" ./fix_pip.sh ./restrict_memory.sh diff --git a/modules/openstack_project/files/nodepool/scripts/prepare_node_tripleo.sh b/modules/openstack_project/files/nodepool/scripts/prepare_node_tripleo.sh index e1d0d83478..a6827db93b 100755 --- a/modules/openstack_project/files/nodepool/scripts/prepare_node_tripleo.sh +++ b/modules/openstack_project/files/nodepool/scripts/prepare_node_tripleo.sh @@ -18,11 +18,11 @@ HOSTNAME=$1 SUDO='true' -BARE='true' +THIN='true' # Workaround bug 1270646 during node bootstrapping. sudo ip link set mtu 1458 dev eth0 -./prepare_node.sh "$HOSTNAME" "$SUDO" "$BARE" +./prepare_node.sh "$HOSTNAME" "$SUDO" "$THIN" sudo -u jenkins -i /opt/nodepool-scripts/prepare_tripleo.sh $HOSTNAME sync diff --git a/modules/openstack_project/files/slave_scripts/baremetal-archive-logs.sh b/modules/openstack_project/files/slave_scripts/baremetal-archive-logs.sh new file mode 100755 index 0000000000..cf3159b868 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/baremetal-archive-logs.sh @@ -0,0 +1,5 @@ +#!/bin/bash -xe + +for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do + cp /var/log/orchestra/rsyslog/$host/syslog $WORKSPACE/logs/$host-syslog.txt +done diff --git a/modules/openstack_project/files/slave_scripts/baremetal-deploy.sh b/modules/openstack_project/files/slave_scripts/baremetal-deploy.sh new file mode 100755 index 0000000000..0421529e3d --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/baremetal-deploy.sh @@ -0,0 +1,44 @@ +#!/bin/bash -xe + +WORKSPACE=`pwd` +mkdir -p logs +rm -f logs/* +cd `dirname "$0"` + +echo "Jenkins: resetting hosts..." +for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do + scp lvm-kexec-reset.sh root@$host:/var/tmp/ + ssh root@$host /var/tmp/lvm-kexec-reset.sh + sudo rm -f /var/log/orchestra/rsyslog/$host/syslog +done + +# Have rsyslog reopen log files we rm'd from under it +sudo restart rsyslog + +# wait for the host to come up (2 ping responses or timeout after 5 minutes) +echo "Jenkins: Waiting for head host to return after reset..." +sleep 10 +if ! timeout 300 ./ping.py $HEAD_HOST; then + echo "Jenkins: ERROR: Head node did not come back up after reset" + exit 1 +fi + +echo "Jenkins: Pre-populating PIP cache" +for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do + scp -r ~/cache/pip root@$host:/var/cache/pip +done + +echo "Jenkins: Caching images." +cd ~/devstack +source stackrc +for image_url in ${IMAGE_URLS//,/ }; do + # Downloads the image (uec ami+aki style), then extracts it. + IMAGE_FNAME=`echo "$image_url" | python -c "import sys; print sys.stdin.read().split('/')[-1]"` + IMAGE_NAME=`echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]"` + if [ ! -f files/$IMAGE_FNAME ]; then + wget -c $image_url -O files/$IMAGE_FNAME + fi +done + +echo "Jenkins: Executing build_bm_multi.sh." +./tools/build_bm_multi.sh diff --git a/modules/openstack_project/files/slave_scripts/baremetal-os-install.sh b/modules/openstack_project/files/slave_scripts/baremetal-os-install.sh new file mode 100755 index 0000000000..4511be0f6c --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/baremetal-os-install.sh @@ -0,0 +1,22 @@ +#!/bin/bash -xe + +set -x +sudo cobbler sync +sudo cobbler system edit --netboot-enabled=Y --name=baremetal1 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal2 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal3 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal4 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal5 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal6 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal7 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal8 +sudo cobbler system edit --netboot-enabled=Y --name=baremetal9 +sudo cobbler system reboot --name=baremetal1 +sudo cobbler system reboot --name=baremetal2 +sudo cobbler system reboot --name=baremetal3 +sudo cobbler system reboot --name=baremetal4 +sudo cobbler system reboot --name=baremetal5 +sudo cobbler system reboot --name=baremetal6 +sudo cobbler system reboot --name=baremetal7 +sudo cobbler system reboot --name=baremetal8 +sudo cobbler system reboot --name=baremetal9 diff --git a/modules/openstack_project/files/slave_scripts/bump-milestone.sh b/modules/openstack_project/files/slave_scripts/bump-milestone.sh new file mode 100644 index 0000000000..8ee90b1888 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/bump-milestone.sh @@ -0,0 +1,56 @@ +#!/bin/bash -xe + +# This script store release meta information in the git repository for +# a project. It does so on an isolated, hidden branch called +# refs/meta/openstack/release. Because it's not under refs/heads, a +# standard clone won't retrieve it or cause it to show up in the list +# of remote branches. The branch shares no history witht the project +# itself; it starts with its own root commit. Jenkins is permitted to +# push directly to refs/meta/openstack/*. + +GIT_HOST="review.openstack.org:29418" +PROJECT_PREFIX="openstack" + +if [[ ! -e ${PROJECT} ]]; then + git clone ssh://$GIT_HOST/$PROJECT_PREFIX/$PROJECT +fi +cd $PROJECT +git checkout master + +# Get the list of meta refs +git fetch origin +refs/meta/*:refs/remotes/meta/* + +# Checkout or create the meta/openstack/release branch +if ! { git branch -a |grep ^[[:space:]]*remotes/meta/openstack/release$; } +then + git checkout --orphan release + # Delete everything so the first commit is truly empty: + git rm -rf . + # git rm -rf leaves submodule directories: + find -maxdepth 1 -not -regex '\./\.git\(/.*\)?' -not -name . -exec rm -fr {} \; + ls -la +else + git branch -D release || /bin/true + git checkout -b release remotes/meta/openstack/release +fi + +# Normally a branch name will just be a file, but we can have branches +# like stable/diablo, so in that case, make the "stable/" directory +# if needed: +mkdir -p `dirname $BRANCH` + +# Read and update the value for the branch +if [ -e "$BRANCH" ] +then + echo "Current contents of ${BRANCH}:" + cat "${BRANCH}" +else + echo "${BRANCH} does not exist. Creating it." +fi + +echo "Updating ${BRANCH} to read $VALUE" +echo "$VALUE" > ${BRANCH} +git add ${BRANCH} + +git commit -m "Milestone ${BRANCH} set to $VALUE" +git push origin HEAD:refs/meta/openstack/release diff --git a/modules/openstack_project/files/slave_scripts/create-ppa-package.sh b/modules/openstack_project/files/slave_scripts/create-ppa-package.sh new file mode 100755 index 0000000000..49b2376c6e --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/create-ppa-package.sh @@ -0,0 +1,78 @@ +#!/bin/bash -xe + +if [ -z "$PROJECT" ] +then + echo '$PROJECT not set.' + exit 1 +fi + +case "$ZUUL_REFNAME" in + master) + export PPAS="ppa:openstack-ppa/bleeding-edge" + ;; + milestone-proposed) + export PPAS="ppa:openstack-ppa/milestone-proposed" + ;; + *) + echo "No PPA defined for branch $ZUUL_REFNAME" + exit 0 +esac + +HUDSON=http://localhost:8080/ +# We keep packaging for openstack trunk in lp:~o-u-p/$project/ubuntu +# For a release (diablo, essex), it's in lp:~o-u-p/$project/$release +OPENSTACK_RELEASE=${OPENSTACK_RELEASE:-ubuntu} +BZR_BRANCH=${BZR_BRANCH:-lp:~openstack-ubuntu-packagers/$PROJECT/${OPENSTACK_RELEASE}} +PPAS=${PPAS:-ppa:$PROJECT-core/trunk} +PACKAGING_REVNO=${PACKAGING_REVNO:--1} +series=${series:-lucid} + +cd build + +tarball="$(echo dist/$PROJECT*.tar.gz)" +version="${tarball%.tar.gz}" +version="${version#*$PROJECT-}" +base_version=$version +if [ -n "${EXTRAVERSION}" ] +then + version="${version%~*}${EXTRAVERSION}~${version#*~}" +fi +tar xvzf "${tarball}" +echo ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz" +ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz" + +# Overlay packaging +# (Intentionally using the natty branch. For these PPA builds, we don't need to diverge +# (yet, at least), so it makes the branch management easier this way. +# Note: Doing a checkout and deleting .bzr afterwards instead of just doing an export, +# because export refuses to overlay over an existing directory, so this was easier. +# (We need to not have the .bzr in there, otherwise vcsversion.py might get overwritten) +echo bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-* +bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-* +cd $PROJECT-* +if [ -d .git ] +then + PACKAGING_REVNO="$(git log --oneline | wc -l)" + rm -rf .git +else + PACKAGING_REVNO="$(bzr revno --tree)" + rm -rf .bzr +fi + +# Please don't change this. It's the only way I'll get notified +# if an upload fails. +export DEBFULLNAME="Soren Hansen" +export DEBEMAIL="soren@openstack.org" + +buildno=$BUILD_NUMBER +pkgversion="${version}-0ubuntu0~${series}${buildno}" +dch -b --force-distribution --v "${pkgversion}" "Automated PPA build. Packaging revision: ${PACKAGING_REVNO}." -D $series +dpkg-buildpackage -rfakeroot -S -sa -nc -k32EE128C +if ! [ "$DO_UPLOAD" = "no" ] +then + for ppa in $PPAS + do + dput --force $ppa "../${PROJECT}_${pkgversion}_source.changes" + done +fi +cd .. diff --git a/modules/openstack_project/files/slave_scripts/docbook-properties.sh b/modules/openstack_project/files/slave_scripts/docbook-properties.sh new file mode 100755 index 0000000000..2b0e31a577 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/docbook-properties.sh @@ -0,0 +1,28 @@ +#!/bin/bash -ex + +# Documentation is published to a URL depending on the branch of the +# openstack-manuals project. This script determines what that location +# should be, and writes a properties file. This is used by Jenkins when +# invoking certain docs jobs and made available to maven. + +# In case we start doing something more sophisticated with other refs +# later (such as tags). +BRANCH=$ZUUL_REFNAME + +# The master branch should get published to /trunk +if [[ $BRANCH == "master" ]]; then + DOC_RELEASE_PATH="trunk" + DOC_COMMENTS_ENABLED=0 +elif [[ $BRANCH =~ ^stable/(.*)$ ]]; then + # The stable/ branch should get published to /releasename, such as icehouse or havana + DOC_RELEASE_PATH=${BASH_REMATCH[1]} + DOC_COMMENTS_ENABLED=1 +else + echo "Error: Branch($BRANCH) is invalid" + exit 1 +fi + +echo "DOC_RELEASE_PATH=$DOC_RELEASE_PATH" >gerrit-doc.properties +echo "DOC_COMMENTS_ENABLED=$DOC_COMMENTS_ENABLED" >>gerrit-doc.properties + +pwd diff --git a/modules/openstack_project/files/slave_scripts/functions.sh b/modules/openstack_project/files/slave_scripts/functions.sh new file mode 100755 index 0000000000..ef013eb2ce --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/functions.sh @@ -0,0 +1,49 @@ +# +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# helper functions + +function check_variable_org_project() +{ + org=$1 + project=$2 + filename=$3 + + if [[ -z "$org" || -z "$project" ]] + then + echo "Usage: $filename ORG PROJECT" + echo + echo "ORG: The project organization (eg 'openstack')" + echo "PROJECT: The project name (eg 'nova')" + exit 1 + fi +} + +function check_variable_version_org_project() +{ + version=$1 + org=$2 + project=$3 + filename=$4 + if [[ -z "$version" || -z "$org" || -z "$project" ]] + then + echo "Usage: $filename VERSION ORG PROJECT" + echo + echo "VERSION: The tox environment python version (eg '27')" + echo "ORG: The project organization (eg 'openstack')" + echo "PROJECT: The project name (eg 'nova')" + exit 1 + fi +} diff --git a/modules/openstack_project/files/slave_scripts/gerrit-git-prep.sh b/modules/openstack_project/files/slave_scripts/gerrit-git-prep.sh new file mode 100755 index 0000000000..8b0c8b86fb --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/gerrit-git-prep.sh @@ -0,0 +1,97 @@ +#!/bin/bash -e + +GERRIT_SITE=$1 +GIT_ORIGIN=$2 + +if [ -z "$GERRIT_SITE" ] +then + echo "The gerrit site name (eg 'https://review.openstack.org') must be the first argument." + exit 1 +fi + +if [ -z "$ZUUL_URL" ] +then + echo "The ZUUL_URL must be provided." + exit 1 +fi + +if [ -z "$GIT_ORIGIN" ] || [ -n "$ZUUL_NEWREV" ] +then + GIT_ORIGIN="$GERRIT_SITE/p" + # git://git.openstack.org/ + # https://review.openstack.org/p +fi + +if [ -z "$ZUUL_REF" ] +then + if [ -n "$BRANCH" ] + then + echo "No ZUUL_REF so using requested branch $BRANCH from origin." + ZUUL_REF=$BRANCH + # use the origin since zuul mergers have outdated branches + ZUUL_URL=$GIT_ORIGIN + else + echo "Provide either ZUUL_REF or BRANCH in the calling enviromnent." + exit 1 + fi +fi + +if [ ! -z "$ZUUL_CHANGE" ] +then + echo "Triggered by: $GERRIT_SITE/$ZUUL_CHANGE" +fi + +set -x +if [[ ! -e .git ]] +then + ls -a + rm -fr .[^.]* * + if [ -d /opt/git/$ZUUL_PROJECT/.git ] + then + git clone file:///opt/git/$ZUUL_PROJECT . + else + git clone $GIT_ORIGIN/$ZUUL_PROJECT . + fi +fi +git remote set-url origin $GIT_ORIGIN/$ZUUL_PROJECT + +# attempt to work around bugs 925790 and 1229352 +if ! git remote update +then + echo "The remote update failed, so garbage collecting before trying again." + git gc + git remote update +fi + +git reset --hard +if ! git clean -x -f -d -q ; then + sleep 1 + git clean -x -f -d -q +fi + +if echo "$ZUUL_REF" | grep -q ^refs/tags/ +then + git fetch --tags $ZUUL_URL/$ZUUL_PROJECT + git checkout $ZUUL_REF + git reset --hard $ZUUL_REF +elif [ -z "$ZUUL_NEWREV" ] +then + git fetch $ZUUL_URL/$ZUUL_PROJECT $ZUUL_REF + git checkout FETCH_HEAD + git reset --hard FETCH_HEAD +else + git checkout $ZUUL_NEWREV + git reset --hard $ZUUL_NEWREV +fi + +if ! git clean -x -f -d -q ; then + sleep 1 + git clean -x -f -d -q +fi + +if [ -f .gitmodules ] +then + git submodule init + git submodule sync + git submodule update --init +fi diff --git a/modules/openstack_project/files/slave_scripts/jenkins-oom-grep.sh b/modules/openstack_project/files/slave_scripts/jenkins-oom-grep.sh new file mode 100755 index 0000000000..b60fa6315c --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/jenkins-oom-grep.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Find out if jenkins has triggered the out-of-memory killer by checking +# the output of dmesg before and after a test run. + +PATTERN=" invoked oom-killer: " + +case "$1" in + pre) + rm -fr /tmp/jenkins-oom-log + mkdir /tmp/jenkins-oom-log + dmesg > /tmp/jenkins-oom-log/pre + exit 0 + ;; + post) + dmesg > /tmp/jenkins-oom-log/post + diff /tmp/jenkins-oom-log/{pre,post} \ + | grep "^> " | sed "s/^> //" > /tmp/jenkins-oom-log/diff + if grep -q "$PATTERN" /tmp/jenkins-oom-log/diff + then + cat /tmp/jenkins-oom-log/diff + exit 1 + fi + ;; +esac diff --git a/modules/openstack_project/files/slave_scripts/jenkins-sudo-grep.sh b/modules/openstack_project/files/slave_scripts/jenkins-sudo-grep.sh new file mode 100755 index 0000000000..4b6e4c319a --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/jenkins-sudo-grep.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Find out if jenkins has attempted to run any sudo commands by checking +# the auth.log or secure log files before and after a test run. + +case $( facter osfamily ) in + Debian) + PATTERN="sudo.*jenkins.*:.*incorrect password attempts" + OLDLOGFILE=/var/log/auth.log.1 + LOGFILE=/var/log/auth.log + ;; + RedHat) + PATTERN="sudo.*jenkins.*:.*command not allowed" + OLDLOGFILE=$( ls /var/log/secure-* | sort | tail -n1 ) + LOGFILE=/var/log/secure + ;; +esac + +case "$1" in + pre) + rm -fr /tmp/jenkins-sudo-log + mkdir /tmp/jenkins-sudo-log + if [ -f $OLDLOGFILE ] + then + stat -c %Y $OLDLOGFILE > /tmp/jenkins-sudo-log/mtime-pre + else + echo "0" > /tmp/jenkins-sudo-log/mtime-pre + fi + grep -h "$PATTERN" $LOGFILE > /tmp/jenkins-sudo-log/pre + exit 0 + ;; + post) + if [ -f $OLDLOGFILE ] + then + stat -c %Y $OLDLOGFILE > /tmp/jenkins-sudo-log/mtime-post + else + echo "0" > /tmp/jenkins-sudo-log/mtime-post + fi + if ! diff /tmp/jenkins-sudo-log/mtime-pre /tmp/jenkins-sudo-log/mtime-post > /dev/null + then + echo "diff" + grep -h "$PATTERN" $OLDLOGFILE > /tmp/jenkins-sudo-log/post + fi + grep -h "$PATTERN" $LOGFILE >> /tmp/jenkins-sudo-log/post + diff /tmp/jenkins-sudo-log/pre /tmp/jenkins-sudo-log/post + ;; +esac diff --git a/modules/openstack_project/files/slave_scripts/jenkinsci-upload.sh b/modules/openstack_project/files/slave_scripts/jenkinsci-upload.sh new file mode 100644 index 0000000000..485c343ed5 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/jenkinsci-upload.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Upload java binaries to maven repositories + +PROJECT=$1 +VERSION=$2 +META_DATA_FILE=$3 +PLUGIN_FILE=$4 + +# Strip project name and extension leaving only the version. +VERSION=`echo ${PLUGIN_FILE} | sed -n "s/${PROJECT}-\(.*\).hpi/\1/p"` + +# generate pom file with version info +POM_IN_ZIP=`unzip -Z -1 ${PLUGIN_FILE}|grep pom.xml` +unzip -o -j ${PLUGIN_FILE} ${POM_IN_ZIP} +sed "s/\${{project-version}}/${VERSION}/g" ${META_DATA_FILE} + +# deploy plugin artifacts from workspace to repo.jenkins-ci.org +JENKINSCI_REPO="http://repo.jenkins-ci.org/list/releases/org/jenkins-ci/plugins" +JENKINSCI_REPO_CREDS="/home/jenkins/.jenkinsci-curl" + +curl -X PUT \ + --config ${JENKINSCI_REPO_CREDS} \ + --data-binary @${META_DATA_FILE} \ + -i "${JENKINSCI_REPO}/${PROJECT}/${VERSION}/${META_DATA_FILE}" > /dev/null 2>&1 + +curl -X PUT \ + --config ${JENKINSCI_REPO_CREDS} \ + --data-binary @${PLUGIN_FILE} \ + -i "${JENKINSCI_REPO}/${PROJECT}/${VERSION}/${PLUGIN_FILE}" > /dev/null 2>&1 + +exit $? diff --git a/modules/openstack_project/files/slave_scripts/lvm-kexec-reset.sh b/modules/openstack_project/files/slave_scripts/lvm-kexec-reset.sh new file mode 100755 index 0000000000..0916b05018 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/lvm-kexec-reset.sh @@ -0,0 +1,7 @@ +#!/bin/bash -x +lvremove -f /dev/main/last_root +lvrename /dev/main/root last_root +lvcreate -L20G -s -n root /dev/main/orig_root +APPEND="`cat /proc/cmdline`" +kexec -l /vmlinuz --initrd=/initrd.img --append="$APPEND" +nohup bash -c "sleep 2; kexec -e" /dev/null 2>&1 & diff --git a/modules/openstack_project/files/slave_scripts/markdown-docbook.sh b/modules/openstack_project/files/slave_scripts/markdown-docbook.sh new file mode 100755 index 0000000000..f037e91466 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/markdown-docbook.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex + +# Documentation can be submitted in markdown and then converted to docbook +# so it can be built with the maven plugin. This is used by Jenkins when +# invoking certain docs jobs and the resulting output is made available to maven. + +# In case we start doing something more sophisticated with other refs +# later (such as tags). +BRANCH=$ZUUL_REFNAME +shopt -s extglob + +# Need to get the file name to insert here so it can be reused for multiple projects +# Filenames for the known repos that could do this are openstackapi-programming.mdown +# and images-api-v2.0.md and openstackapi-programming and images-api-v2.0 are the names +# for the ID and xml filename. +FILENAME=$1 +FILEPATH=`find ./ -regextype posix-extended -regex ".*${FILENAME}\.(md|markdown|mdown)"` +DIRPATH=`dirname $FILEPATH` +pandoc -f markdown -t docbook -s ${FILEPATH} | xsltproc -o - /usr/share/xml/docbook/stylesheet/docbook5/db4-upgrade.xsl - | xmllint --format - | sed -e "s,,,' > ${DIRPATH}/$FILENAME.xml + +pwd diff --git a/modules/openstack_project/files/slave_scripts/mavencentral-upload.sh b/modules/openstack_project/files/slave_scripts/mavencentral-upload.sh new file mode 100644 index 0000000000..cd162ecf65 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/mavencentral-upload.sh @@ -0,0 +1,44 @@ +#!/bin/bash -x +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Upload java packages to maven repositories + +PROJECT=$1 +VERSION=$2 +META_DATA_FILE=$3 +PLUGIN_FILE=$4 + +# Strip project name and extension leaving only the version. +VERSION=`echo ${PLUGIN_FILE} | sed -n "s/${PROJECT}-\(.*\).jar/\1/p"` + +# generate pom file with version info +POM_IN_ZIP=`unzip -Z -1 ${PLUGIN_FILE}|grep pom.xml` +unzip -o -j ${PLUGIN_FILE} ${POM_IN_ZIP} +sed "s/\${{project-version}}/${VERSION}/g" ${META_DATA_FILE} + +# deploy plugin artifacts from workspace to maven central repository +MAVEN_REPO="https://oss.sonatype.org/content/groups/public/maven" +MAVEN_REPO_CREDS="/home/jenkins/.mavencentral-curl" + +curl -X PUT \ + --config ${MAVEN_REPO_CREDS} \ + --data-binary @${META_DATA_FILE} \ + -i "${MAVEN_REPO}/${PROJECT}/${VERSION}/${META_DATA_FILE}" > /dev/null 2>&1 + +curl -X PUT \ + --config ${MAVEN_REPO_CREDS} \ + --data-binary @${PLUGIN_FILE} \ + -i "${MAVEN_REPO}/${PROJECT}/${VERSION}/${PLUGIN_FILE}" > /dev/null 2>&1 diff --git a/modules/openstack_project/files/slave_scripts/merge_tags.sh b/modules/openstack_project/files/slave_scripts/merge_tags.sh new file mode 100755 index 0000000000..dad31e63dd --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/merge_tags.sh @@ -0,0 +1,30 @@ +#!/bin/bash -xe + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +TAG=$1 + +if $(git tag --contains origin/milestone-proposed | grep "^$TAG$" >/dev/null) +then + git config user.name "OpenStack Proposal Bot" + git config user.email "openstack-infra@lists.openstack.org" + git config gitreview.username "proposal-bot" + + git review -s + git checkout master + git reset --hard origin/master + git merge --no-edit -s ours $TAG + # Get a Change-Id + GIT_EDITOR=true git commit --amend + git review -R -y -t merge/release-tag +fi diff --git a/modules/openstack_project/files/slave_scripts/package-gerrit.sh b/modules/openstack_project/files/slave_scripts/package-gerrit.sh new file mode 100644 index 0000000000..1254ce4e81 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/package-gerrit.sh @@ -0,0 +1,7 @@ +#!/bin/bash -xe + +rm -fr ~/.m2 +rm -fr ~/.java +./tools/version.sh --release +mvn clean package -Dgerrit.include-documentation=1 -X +./tools/version.sh --reset diff --git a/modules/openstack_project/files/slave_scripts/php-laravel-build.sh b/modules/openstack_project/files/slave_scripts/php-laravel-build.sh new file mode 100644 index 0000000000..f8e6fca59a --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/php-laravel-build.sh @@ -0,0 +1,12 @@ +#!/bin/bash -xe +# Build a Laravel/PHP distribution using composer. + +cat >bootstrap/environment.php <detectEnvironment(function() +{ + return 'dev'; +}); +EOF +curl -s https://getcomposer.org/installer | /usr/bin/php +php composer.phar install --prefer-dist \ No newline at end of file diff --git a/modules/openstack_project/files/slave_scripts/ping.py b/modules/openstack_project/files/slave_scripts/ping.py new file mode 100755 index 0000000000..ad4474aaf9 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/ping.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +import sys +from subprocess import Popen, PIPE + +p = Popen(["ping", sys.argv[1]], stdout=PIPE) +while True: + line = p.stdout.readline().strip() + if 'bytes from' in line: + p.terminate() + sys.exit(0) diff --git a/modules/openstack_project/files/slave_scripts/project-requirements-change.py b/modules/openstack_project/files/slave_scripts/project-requirements-change.py new file mode 100755 index 0000000000..996668cf25 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/project-requirements-change.py @@ -0,0 +1,155 @@ +#! /usr/bin/env python +# Copyright (C) 2011 OpenStack, LLC. +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# Copyright (c) 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import pkg_resources +import shlex +import shutil +import subprocess +import sys +import tempfile + + +def run_command(cmd): + print(cmd) + cmd_list = shlex.split(str(cmd)) + p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + (out, nothing) = p.communicate() + return out.strip() + + +class RequirementsList(object): + def __init__(self, name): + self.name = name + self.reqs = {} + self.failed = False + + def read_requirements(self, fn, ignore_dups=False, strict=False): + """ Read a requirements file and optionally enforce style.""" + if not os.path.exists(fn): + return + for line in open(fn): + if strict and '\n' not in line: + raise Exception("Requirements file %s does not " + "end with a newline." % fn) + if '#' in line: + line = line[:line.find('#')] + line = line.strip() + if (not line or + line.startswith('http://tarballs.openstack.org/') or + line.startswith('-e') or + line.startswith('-f')): + continue + req = pkg_resources.Requirement.parse(line) + if (not ignore_dups and strict and req.project_name.lower() + in self.reqs): + print("Duplicate requirement in %s: %s" % + (self.name, str(req))) + self.failed = True + self.reqs[req.project_name.lower()] = req + + def read_all_requirements(self, global_req=False, include_dev=False, + strict=False): + """ Read all the requirements into a list. + + Build ourselves a consolidated list of requirements. If global_req is + True then we are parsing the global requirements file only, and + ensure that we don't parse it's test-requirements.txt erroneously. + + If include_dev is true allow for development requirements, which + may be prereleased versions of libraries that would otherwise be + listed. This is most often used for olso prereleases. + + If strict is True then style checks should be performed while reading + the file. + """ + if global_req: + self.read_requirements('global-requirements.txt', strict=strict) + else: + for fn in ['tools/pip-requires', + 'tools/test-requires', + 'requirements.txt', + 'test-requirements.txt' + ]: + self.read_requirements(fn, strict=strict) + if include_dev: + self.read_requirements('dev-requirements.txt', + ignore_dups=True, strict=strict) + + +def main(): + branch = sys.argv[1] + + # build a list of requirements in the proposed change, + # and check them for style violations while doing so + head = run_command("git rev-parse HEAD").strip() + head_reqs = RequirementsList('HEAD') + head_reqs.read_all_requirements(strict=True) + + # build a list of requirements already in the target branch, + # so that we can create a diff and identify what's being changed + run_command("git remote update") + run_command("git checkout remotes/origin/%s" % branch) + branch_reqs = RequirementsList(branch) + branch_reqs.read_all_requirements() + + # switch back to the proposed change now + run_command("git checkout %s" % head) + + # build a list of requirements from the global list in the + # openstack/requirements project so we can match them to the changes + reqroot = tempfile.mkdtemp() + reqdir = os.path.join(reqroot, "requirements") + run_command("git clone https://review.openstack.org/p/openstack/" + "requirements --depth 1 %s" % reqdir) + os.chdir(reqdir) + run_command("git checkout remotes/origin/%s" % branch) + print "requirements git sha: %s" % run_command( + "git rev-parse HEAD").strip() + os_reqs = RequirementsList('openstack/requirements') + os_reqs.read_all_requirements(include_dev=(branch == 'master'), + global_req=True) + + # iterate through the changing entries and see if they match the global + # equivalents we want enforced + failed = False + for req in head_reqs.reqs.values(): + name = req.project_name.lower() + if name in branch_reqs.reqs and req == branch_reqs.reqs[name]: + continue + if name not in os_reqs.reqs: + print("Requirement %s not in openstack/requirements" % str(req)) + failed = True + continue + # pkg_resources.Requirement implements __eq__() but not __ne__(). + # There is no implied relationship between __eq__() and __ne__() + # so we must negate the result of == here instead of using !=. + if not (req == os_reqs.reqs[name]): + print("Requirement %s does not match openstack/requirements " + "value %s" % (str(req), str(os_reqs.reqs[name]))) + failed = True + + # clean up and report the results + shutil.rmtree(reqroot) + if failed or os_reqs.failed or head_reqs.failed or branch_reqs.failed: + sys.exit(1) + print("Updated requirements match openstack/requirements.") + + +if __name__ == '__main__': + main() diff --git a/modules/openstack_project/files/slave_scripts/propose_requirements_update.sh b/modules/openstack_project/files/slave_scripts/propose_requirements_update.sh new file mode 100755 index 0000000000..0fb7115de9 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/propose_requirements_update.sh @@ -0,0 +1,97 @@ +#!/bin/bash -xe + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +INITIAL_COMMIT_MSG="Updated from global requirements" +TOPIC="openstack/requirements" +USERNAME="proposal-bot" +BRANCH=$ZUUL_REF +ALL_SUCCESS=0 + +if [ -z "$BRANCH" ] ; then + echo "No branch set, exiting." + exit 1 +fi + +git config user.name "OpenStack Proposal Bot" +git config user.email "openstack-infra@lists.openstack.org" +git config gitreview.username "proposal-bot" + +for PROJECT in $(cat projects.txt); do + + change_id="" + # See if there is an open change in the openstack/requirements topic + # If so, get the change id for the existing change for use in the + # commit msg. + change_info=$(ssh -p 29418 $USERNAME@review.openstack.org gerrit query --current-patch-set status:open project:$PROJECT topic:$TOPIC owner:$USERNAME branch:$BRANCH) + previous=$(echo "$change_info" | grep "^ number:" | awk '{print $2}') + if [ "x${previous}" != "x" ] ; then + change_id=$(echo "$change_info" | grep "^change" | awk '{print $2}') + # read return a non zero value when it reaches EOF. Because we use a + # heredoc here it will always reach EOF and return a nonzero value. + # Disable -e temporarily to get around the read. + # The reason we use read is to allow for multiline variable content + # and variable interpolation. Simply double quoting a string across + # multiple lines removes the newlines. + set +e + read -d '' COMMIT_MSG </LC_MESSAGES/${PROJECT}.po" --source-lang en --source-file ${PROJECT}/locale/${PROJECT}.pot -t PO --execute + +# Pull upstream translations of files that are at least 75 % +# translated +tx pull -a -f --minimum-perc=75 + +# Update the .pot file +python setup.py extract_messages +PO_FILES=`find ${PROJECT}/locale -name '*.po'` +if [ -n "$PO_FILES" ] +then + # Use updated .pot file to update translations + python setup.py update_catalog --no-fuzzy-matching --ignore-obsolete=true +fi +# Add all changed files to git +git add $PROJECT/locale/* + +# Don't send files where the only things which have changed are the +# creation date, the version number, the revision date, or comment +# lines. +for f in `git diff --cached --name-only` +do + if [ `git diff --cached $f |egrep -v "(POT-Creation-Date|Project-Id-Version|PO-Revision-Date|^\+{3}|^\-{3}|^[-+]#)" | egrep -c "^[\-\+]"` -eq 0 ] + then + git reset -q $f + git checkout -- $f + fi +done + +# Don't send a review if nothing has changed. +if [ `git diff --cached |wc -l` -gt 0 ] +then + # Commit and review + git commit -F- </LC_MESSAGES/djangojs.po" --source-lang en \ +--source-file ${PROJECT}/locale/en/LC_MESSAGES/djangojs.po -t PO --execute +# Horizon Translations +tx set --auto-local -r ${PROJECT}.${PROJECT}-translations \ +"${PROJECT}/locale//LC_MESSAGES/django.po" --source-lang en \ +--source-file ${PROJECT}/locale/en/LC_MESSAGES/django.po -t PO --execute +# OpenStack Dashboard Translations +tx set --auto-local -r ${PROJECT}.openstack-dashboard-translations \ +"openstack_dashboard/locale//LC_MESSAGES/django.po" --source-lang en \ +--source-file openstack_dashboard/locale/en/LC_MESSAGES/django.po -t PO --execute + +# Pull upstream translations of files that are at least 75 % +# translated +tx pull -a -f --minimum-perc=75 + +# Invoke run_tests.sh to update the po files +# Or else, "../manage.py makemessages" can be used. +./run_tests.sh --makemessages -V + +# Add all changed files to git +git add horizon/locale/* openstack_dashboard/locale/* + +# Don't send files where the only things which have changed are the +# creation date, the version number, the revision date, or comment +# lines. +for f in `git diff --cached --name-only` +do + if [ `git diff --cached $f |egrep -v "(POT-Creation-Date|Project-Id-Version|PO-Revision-Date|^\+{3}|^\-{3}|^[-+]#)" | egrep -c "^[\-\+]"` -eq 0 ] + then + git reset -q $f + git checkout -- $f + fi +done + +# Don't send a review if nothing has changed. +if [ `git diff --cached |wc -l` -gt 0 ] +then + # Commit and review + git commit -F- < ${DocFolder}/high-availability-guide/bk-ha-guide.xml + fi + # Update the .pot file + ./tools/generatepot ${DOCNAME} + if [ -f ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ] + then + # Add all changed files to git + git add ${DocFolder}/${DOCNAME}/locale/* + # Set auto-local + tx set --auto-local -r openstack-manuals-i18n.${DOCNAME} \ +"${DocFolder}/${DOCNAME}/locale/.po" --source-lang en \ +--source-file ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot \ +-t PO --execute + fi +done + +# Pull upstream translations of files that are at least 75 % +# translated +tx pull -a -f --minimum-perc=75 + +# The common directory is used by the other guides, let's be more +# liberal here since teams might only translate the files used by a +# single guide. We use 8 % since that downloads the currently +# translated files. +if [ $PROJECT = "openstack-manuals" ] ; then + tx pull -f --minimum-perc=8 -r openstack-manuals-i18n.common +fi + + +for FILE in ${DocFolder}/* +do + DOCNAME=${FILE#${DocFolder}/} + if [ -d ${DocFolder}/${DOCNAME}/locale ] + then + git add ${DocFolder}/${DOCNAME}/locale/* + fi +done + +# Don't send files where the only things which have changed are the +# creation date, the version number, the revision date, or comment +# lines. +for f in `git diff --cached --name-only` +do + if [ `git diff --cached $f |egrep -v "(POT-Creation-Date|Project-Id-Version|PO-Revision-Date|^\+{3}|^\-{3}|^[-+]#)" | egrep -c "^[\-\+]"` -eq 0 ] + then + git reset -q $f + git checkout -- $f + fi +done + +# Don't send a review if nothing has changed. +if [ `git diff --cached |wc -l` -gt 0 ] +then + # Commit and review + git commit -F- </dev/null` ; then + # Put tagged releases in proper location. All tagged builds get copied to + # BUILD_DIR/tagname. If this is the latest tagged release the copy of files + # at BUILD_DIR remains. When Jenkins copies this file the root developer + # docs are always the latest release with older tags available under the + # root in the tagname dir. + TAG=`echo $ZUUL_REFNAME | sed 's/refs.tags.//'` + if [ ! -z $TAG ] ; then + if echo $ZUUL_PROJECT | grep 'python-.*client' ; then + # This is a hack to ignore the year.release tags in python-*client + # projects. + LATEST=`git tag | sed -n -e '/^2012\..*$/d' -e '/^\([0-9]\+\.\?\)\+$/p' | sort -V | tail -1` + else + # Take all tags of the form (number.)+, sort them, then take the + # largest + LATEST=`git tag | sed -n '/^\([0-9]\+\.\?\)\+$/p' | sort -V | tail -1` + fi + if [ "$TAG" = "$LATEST" ] ; then + # Copy the docs into a subdir if this is a tagged build + mkdir doc/build/$TAG + cp -R doc/build/html/* doc/build/$TAG + mv doc/build/$TAG doc/build/html/$TAG + else + # Move the docs into a subdir if this is a tagged build + mkdir doc/build/$TAG + mv doc/build/html/* doc/build/$TAG + mv doc/build/$TAG doc/build/html/$TAG + fi + fi +elif `echo $ZUUL_REFNAME | grep stable/ >/dev/null` ; then + # Put stable release changes in dir named after stable release under the + # build dir. When Jenkins copies these files they will be accessible under + # the developer docs root using the stable release's name. + BRANCH=`echo $ZUUL_REFNAME | sed 's/stable.//'` + if [ ! -z $BRANCH ] ; then + # Move the docs into a subdir if this is a stable branch build + mkdir doc/build/$BRANCH + mv doc/build/html/* doc/build/$BRANCH + mv doc/build/$BRANCH doc/build/html/$BRANCH + fi +else + # Put other branch changes in dir named after branch under the + # build dir. When Jenkins copies these files they will be + # accessible under the developer docs root using the branch name. + # EG: feature/foo or milestone-proposed + BRANCH=$ZUUL_REFNAME + mkdir doc/build/tmp + mv doc/build/html/* doc/build/tmp + mkdir -p doc/build/html/$BRANCH + mv doc/build/tmp/* doc/build/html/$BRANCH +fi + +echo "Begin pip freeze output from test virtualenv:" +echo "======================================================================" +.tox/$venv/bin/pip freeze +echo "======================================================================" + +exit $result diff --git a/modules/openstack_project/files/slave_scripts/run-jsbuild.sh b/modules/openstack_project/files/slave_scripts/run-jsbuild.sh new file mode 100755 index 0000000000..8119605374 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/run-jsbuild.sh @@ -0,0 +1,40 @@ +#!/bin/bash -xe +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +command=$1 +org=$2 +project=$3 + +source /usr/local/jenkins/slave_scripts/functions.sh +check_variable_org_project "$org" "$project" "$0" + +source /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project + +rm -f dist/*.tar.gz + +venv=grunt +VDISPLAY=99 +DIMENSIONS='1280x1024x24' + +set +e +/usr/bin/Xvfb :${VDISPLAY} -screen 0 ${DIMENSIONS} 2>&1 > /dev/null & + +DISPLAY=:${VDISPLAY} tox -e$venv $command +result=$? + +pkill Xvfb 2>&1 > /dev/null +set -e + +exit $result diff --git a/modules/openstack_project/files/slave_scripts/run-pep8.sh b/modules/openstack_project/files/slave_scripts/run-pep8.sh new file mode 100755 index 0000000000..fda4aa3909 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/run-pep8.sh @@ -0,0 +1,28 @@ +#!/bin/bash -xe + +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +org=$1 +project=$2 + +source /usr/local/jenkins/slave_scripts/functions.sh +check_variable_org_project "$org" "$project" "$0" + +source /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project + +set -o pipefail +tox -v -epep8 | tee pep8.txt +set +o pipefail + diff --git a/modules/openstack_project/files/slave_scripts/run-pylint.sh b/modules/openstack_project/files/slave_scripts/run-pylint.sh new file mode 100755 index 0000000000..cd6e5a698f --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/run-pylint.sh @@ -0,0 +1,28 @@ +#!/bin/bash -xe + +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +org=$1 +project=$2 + +source /usr/local/jenkins/slave_scripts/functions.sh +check_variable_org_project "$org" "$project" "$0" + +source /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project + +set -o pipefail +tox -v -epylint | tee pylint.txt +set +o pipefail + diff --git a/modules/openstack_project/files/slave_scripts/run-selenium.sh b/modules/openstack_project/files/slave_scripts/run-selenium.sh new file mode 100755 index 0000000000..73541c7c2d --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/run-selenium.sh @@ -0,0 +1,37 @@ +#!/bin/bash -xe + +# If a bundle file is present, call tox with the jenkins version of +# the test environment so it is used. Otherwise, use the normal +# (non-bundle) test environment. Also, run pip freeze on the +# resulting environment at the end so that we have a record of exactly +# what packages we ended up testing. +# + +org=$1 +project=$2 + +source /usr/local/jenkins/slave_scripts/functions.sh +check_variable_org_project "$org" "$project" "$0" + +source /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project + +venv=venv + +VDISPLAY=99 +DIMENSIONS='1280x1024x24' +/usr/bin/Xvfb :${VDISPLAY} -screen 0 ${DIMENSIONS} 2>&1 > /dev/null & + +set +e +DISPLAY=:${VDISPLAY} NOSE_WITH_XUNIT=1 tox -e$venv -- \ + /bin/bash run_tests.sh -N --only-selenium +result=$? + +pkill Xvfb 2>&1 > /dev/null +set -e + +echo "Begin pip freeze output from test virtualenv:" +echo "======================================================================" +.tox/$venv/bin/pip freeze +echo "======================================================================" + +exit $result diff --git a/modules/openstack_project/files/slave_scripts/run-tarball.sh b/modules/openstack_project/files/slave_scripts/run-tarball.sh new file mode 100755 index 0000000000..c022f28811 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/run-tarball.sh @@ -0,0 +1,38 @@ +#!/bin/bash -xe + +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +org=$1 +project=$2 + +source /usr/local/jenkins/slave_scripts/functions.sh +check_variable_org_project "$org" "$project" "$0" + +source /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project + +rm -f dist/*.tar.gz +tox -evenv python setup.py sdist + +FILES=dist/*.tar.gz +for f in $FILES +do + echo "SHA1sum for $f:" + sha1sum $f | awk '{print $1}' > $f.sha1 + cat $f.sha1 + + echo "MD5sum for $f:" + md5sum $f | awk '{print $1}' > $f.md5 + cat $f.md5 +done diff --git a/modules/openstack_project/files/slave_scripts/run-tox.sh b/modules/openstack_project/files/slave_scripts/run-tox.sh new file mode 100755 index 0000000000..afe8092ee7 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/run-tox.sh @@ -0,0 +1,62 @@ +#!/bin/bash -x + +# If a bundle file is present, call tox with the jenkins version of +# the test environment so it is used. Otherwise, use the normal +# (non-bundle) test environment. Also, run pip freeze on the +# resulting environment at the end so that we have a record of exactly +# what packages we ended up testing. +# +# Usage: run-tox.sh VENV +# +# Where VENV is the name of the tox environment to run (specified in the +# project's tox.ini file). + +venv=$1 +org=$2 +project=$3 + +if [[ -z "$venv" || -z "$org" || -z "$project" ]] +then + echo "Usage: $? VENV ORG PROJECT" + echo + echo "VENV: The tox environment to run (eg 'python27')" + echo "ORG: The project organization (eg 'stackforge')" + echo "PROJECT: The project name (eg 'nova')" + exit 1 +fi + +/usr/local/jenkins/slave_scripts/jenkins-oom-grep.sh pre + +sudo /usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh pre + +source /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project + +tox -v -e$venv +result=$? + +sudo /usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh post +sudoresult=$? + +if [ $sudoresult -ne "0" ] +then + echo + echo "This test has failed because it attempted to execute commands" + echo "with sudo. See above for the exact commands used." + echo + exit 1 +fi + +/usr/local/jenkins/slave_scripts/jenkins-oom-grep.sh post +oomresult=$? + +if [ $oomresult -ne "0" ] +then + echo + echo "This test has failed because it attempted to exceed configured" + echo "memory limits and was killed prior to completion. See above" + echo "for related kernel messages." + echo + exit 1 +fi + +exit $result diff --git a/modules/openstack_project/files/slave_scripts/run-unittests.sh b/modules/openstack_project/files/slave_scripts/run-unittests.sh new file mode 100755 index 0000000000..36287d4520 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/run-unittests.sh @@ -0,0 +1,118 @@ +#!/bin/bash -x + +# Call tox with the jenkins version of the test environment so it is used. +# Also, run pip freeze on the resulting environment at the end so that we have +# a record of exactly what packages we ended up testing. +# +# Usage: run-unittests.sh PYTHONVERSION +# +# Where PYTHONVERSION is the numeric version identifier used as a suffix +# in the tox.ini file. E.g., "26" or "27" for "py26"/"jenkins26" or +# "py27"/"jenkins27" respectively. + +version=$1 +org=$2 +project=$3 + +source /usr/local/jenkins/slave_scripts/functions.sh +check_variable_version_org_project "$version" "$org" "$project" "$0" + +venv=py$version + +export NOSE_WITH_XUNIT=1 +export NOSE_WITH_HTML_OUTPUT=1 +export NOSE_HTML_OUT_FILE='nose_results.html' +export TMPDIR=`/bin/mktemp -d` +trap "rm -rf $TMPDIR" EXIT + +/usr/local/jenkins/slave_scripts/jenkins-oom-grep.sh pre + +sudo /usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh pre + +source /usr/local/jenkins/slave_scripts/select-mirror.sh $org $project + +tox -e$venv +result=$? + +echo "Begin pip freeze output from test virtualenv:" +echo "======================================================================" +.tox/$venv/bin/pip freeze +echo "======================================================================" + +if [ -d ".testrepository" ] ; then + if [ -f ".testrepository/0.2" ] ; then + cp .testrepository/0.2 ./subunit_log.txt + elif [ -f ".testrepository/0" ] ; then + .tox/$venv/bin/subunit-1to2 < .testrepository/0 > ./subunit_log.txt + fi + .tox/$venv/bin/python /usr/local/jenkins/slave_scripts/subunit2html.py ./subunit_log.txt testr_results.html + SUBUNIT_SIZE=$(du -k ./subunit_log.txt | awk '{print $1}') + gzip -9 ./subunit_log.txt + gzip -9 ./testr_results.html + + export PYTHON=.tox/$venv/bin/python + if [[ "$SUBUNIT_SIZE" -gt 50000 ]]; then + echo + echo "sub_unit.log was > 50 MB of uncompressed data!!!" + echo "Something is causing tests for this project to log significant amounts" + echo "of data. This may be writers to python logging, stdout, or stderr." + echo "Failing this test as a result" + echo + exit 1 + fi + + rancount=$(.tox/$venv/bin/testr last | sed -ne 's/Ran \([0-9]\+\).*tests in.*/\1/p') + if [ -z "$rancount" ] || [ "$rancount" -eq "0" ] ; then + echo + echo "Zero tests were run. At least one test should have been run." + echo "Failing this test as a result" + echo + exit 1 + fi +fi + +sudo /usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh post +sudoresult=$? + +if [ $sudoresult -ne "0" ] +then + echo + echo "This test has failed because it attempted to execute commands" + echo "with sudo. See above for the exact commands used." + echo + exit 1 +fi + +/usr/local/jenkins/slave_scripts/jenkins-oom-grep.sh post +oomresult=$? + +if [ $oomresult -ne "0" ] +then + echo + echo "This test has failed because it attempted to exceed configured" + echo "memory limits and was killed prior to completion. See above" + echo "for related kernel messages." + echo + exit 1 +fi + +htmlreport=$(find . -name $NOSE_HTML_OUT_FILE) +if [ -f "$htmlreport" ] +then + passcount=$(grep -c 'tr class=.passClass' $htmlreport) + if [ $passcount -eq "0" ] + then + echo + echo "Zero tests passed, which probably means there was an error" + echo "parsing one of the python files, or that some other failure" + echo "during test setup prevented a sane run." + echo + exit 1 + fi +else + echo + echo "WARNING: Unable to find $NOSE_HTML_OUT_FILE to confirm results!" + echo +fi + +exit $result diff --git a/modules/openstack_project/files/slave_scripts/run-xmllint.sh b/modules/openstack_project/files/slave_scripts/run-xmllint.sh new file mode 100755 index 0000000000..4edf7d61bd --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/run-xmllint.sh @@ -0,0 +1,4 @@ +#!/bin/bash -xe + +mkdir -p target/ +/usr/bin/xmllint -noent $1 > target/`basename $1` diff --git a/modules/openstack_project/files/slave_scripts/select-mirror.sh b/modules/openstack_project/files/slave_scripts/select-mirror.sh new file mode 100755 index 0000000000..9ea4598250 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/select-mirror.sh @@ -0,0 +1,57 @@ +#!/bin/bash -x + +# Copyright 2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +org=$1 +project=$2 + +source /usr/local/jenkins/slave_scripts/functions.sh +check_variable_org_project "$org" "$project" "$0" + +rm -f ~/.pydistutils.cfg +mkdir -p ~/.pip +rm -f ~/.pip/pip.conf + +# Start with a default pip.conf for use with pypi.python.org +# (which may be overwritten later) +cat < ~/.pip/pip.conf +[global] +timeout = 60 +EOF + +# For project listed in openstack/requirements, +# use the pypi.openstack.org mirror exclusively +if grep -x "$org/$project" /opt/requirements/projects.txt 2>&1 +then + export TOX_INDEX_URL='http://pypi.openstack.org/openstack' + echo "Switching on internal pypi mirror $TOX_INDEX_URL for $org/$project" + cat < ~/.pydistutils.cfg +[easy_install] +index_url = http://pypi.openstack.org/openstack +EOF + cat < ~/.pip/pip.conf +[global] +index-url = http://pypi.openstack.org/openstack +timeout = 60 +EOF +else + echo "$org/$project will not use the internal openstack pypi mirror" + cat < ~/.pip/pip.conf +[global] +timeout = 60 +index-url = http://pypi.openstack.org/openstack +extra-index-url = http://pypi.python.org/simple +EOF +fi diff --git a/modules/openstack_project/files/slave_scripts/subunit2html.py b/modules/openstack_project/files/slave_scripts/subunit2html.py new file mode 100755 index 0000000000..96c289f4c8 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/subunit2html.py @@ -0,0 +1,727 @@ +#!/usr/bin/python +""" +Utility to convert a subunit stream to an html results file. +Code is adapted from the pyunit Html test runner at +http://tungwaiyip.info/software/HTMLTestRunner.html + +Takes two arguments. First argument is path to subunit log file, second +argument is path of desired output file. Second argument is optional, +defaults to 'results.html'. + +Original HTMLTestRunner License: +------------------------------------------------------------------------ +Copyright (c) 2004-2007, Wai Yip Tung +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name Wai Yip Tung nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +import collections +import datetime +import io +import sys +import traceback +from xml.sax import saxutils + +import subunit +import testtools + +__version__ = '0.1' + + +class TemplateData(object): + """ + Define a HTML template for report customerization and generation. + + Overall structure of an HTML report + + HTML + +------------------------+ + | | + | | + | | + | STYLESHEET | + | +----------------+ | + | | | | + | +----------------+ | + | | + | | + | | + | | + | | + | HEADING | + | +----------------+ | + | | | | + | +----------------+ | + | | + | REPORT | + | +----------------+ | + | | | | + | +----------------+ | + | | + | ENDING | + | +----------------+ | + | | | | + | +----------------+ | + | | + | | + | | + +------------------------+ + """ + + STATUS = { + 0: 'pass', + 1: 'fail', + 2: 'error', + 3: 'skip', + } + + DEFAULT_TITLE = 'Unit Test Report' + DEFAULT_DESCRIPTION = '' + + # ------------------------------------------------------------------------ + # HTML Template + + HTML_TMPL = r""" + + + + %(title)s + + + %(stylesheet)s + + + + +%(heading)s +%(report)s +%(ending)s + + + +""" + # variables: (title, generator, stylesheet, heading, report, ending) + + # ------------------------------------------------------------------------ + # Stylesheet + # + # alternatively use a for external style sheet, e.g. + # + + STYLESHEET_TMPL = """ + +""" + + # ------------------------------------------------------------------------ + # Heading + # + + HEADING_TMPL = """
+

%(title)s

+%(parameters)s +

%(description)s

+
+ +""" # variables: (title, parameters, description) + + HEADING_ATTRIBUTE_TMPL = """ +

%(name)s: %(value)s

+""" # variables: (name, value) + + # ------------------------------------------------------------------------ + # Report + # + + REPORT_TMPL = """ +

Show +Summary +Failed +All +

+ ++++++++++ + + + + + + + + + + +%(test_list)s + + + + + + + + + + +
Test Group/Test caseCountPassFailErrorSkipView
Total%(count)s%(Pass)s%(fail)s%(error)s%(skip)s  
+""" # variables: (test_list, count, Pass, fail, error) + + REPORT_CLASS_TMPL = r""" + + %(desc)s + %(count)s + %(Pass)s + %(fail)s + %(error)s + %(skip)s + Detail + + +""" # variables: (style, desc, count, Pass, fail, error, cid) + + REPORT_TEST_WITH_OUTPUT_TMPL = r""" + +
%(desc)s
+ + + + + %(status)s + + + + + + +""" # variables: (tid, Class, style, desc, status) + + REPORT_TEST_NO_OUTPUT_TMPL = r""" + +
%(desc)s
+ %(status)s + +""" # variables: (tid, Class, style, desc, status) + + REPORT_TEST_OUTPUT_TMPL = r""" +%(id)s: %(output)s +""" # variables: (id, output) + + # ------------------------------------------------------------------------ + # ENDING + # + + ENDING_TMPL = """
 
""" + +# -------------------- The end of the Template class ------------------- + + +class ClassInfoWrapper(object): + def __init__(self, name, mod): + self.name = name + self.mod = mod + + def __repr__(self): + return "%s" % (self.name) + + +class HtmlOutput(testtools.TestResult): + """Output test results in html.""" + + def __init__(self, html_file='result.html'): + super(HtmlOutput, self).__init__() + self.success_count = 0 + self.failure_count = 0 + self.error_count = 0 + self.skip_count = 0 + self.result = [] + self.html_file = html_file + + def addSuccess(self, test): + self.success_count += 1 + output = test.shortDescription() + if output is None: + output = test.id() + self.result.append((0, test, output, '')) + + def addSkip(self, test, err): + output = test.shortDescription() + if output is None: + output = test.id() + self.skip_count += 1 + self.result.append((3, test, output, '')) + + def addError(self, test, err): + output = test.shortDescription() + if output is None: + output = test.id() + # Skipped tests are handled by SkipTest Exceptions. + #if err[0] == SkipTest: + # self.skip_count += 1 + # self.result.append((3, test, output, '')) + else: + self.error_count += 1 + _exc_str = self.formatErr(err) + self.result.append((2, test, output, _exc_str)) + + def addFailure(self, test, err): + print(test) + self.failure_count += 1 + _exc_str = self.formatErr(err) + output = test.shortDescription() + if output is None: + output = test.id() + self.result.append((1, test, output, _exc_str)) + + def formatErr(self, err): + exctype, value, tb = err + return ''.join(traceback.format_exception(exctype, value, tb)) + + def stopTestRun(self): + super(HtmlOutput, self).stopTestRun() + self.stopTime = datetime.datetime.now() + report_attrs = self._getReportAttributes() + generator = 'subunit2html %s' % __version__ + heading = self._generate_heading(report_attrs) + report = self._generate_report() + ending = self._generate_ending() + output = TemplateData.HTML_TMPL % dict( + title=saxutils.escape(TemplateData.DEFAULT_TITLE), + generator=generator, + stylesheet=TemplateData.STYLESHEET_TMPL, + heading=heading, + report=report, + ending=ending, + ) + if self.html_file: + with open(self.html_file, 'wb') as html_file: + html_file.write(output.encode('utf8')) + + def _getReportAttributes(self): + """Return report attributes as a list of (name, value).""" + status = [] + if self.success_count: + status.append('Pass %s' % self.success_count) + if self.failure_count: + status.append('Failure %s' % self.failure_count) + if self.error_count: + status.append('Error %s' % self.error_count) + if self.skip_count: + status.append('Skip %s' % self.skip_count) + if status: + status = ' '.join(status) + else: + status = 'none' + return [ + ('Status', status), + ] + + def _generate_heading(self, report_attrs): + a_lines = [] + for name, value in report_attrs: + line = TemplateData.HEADING_ATTRIBUTE_TMPL % dict( + name=saxutils.escape(name), + value=saxutils.escape(value), + ) + a_lines.append(line) + heading = TemplateData.HEADING_TMPL % dict( + title=saxutils.escape(TemplateData.DEFAULT_TITLE), + parameters=''.join(a_lines), + description=saxutils.escape(TemplateData.DEFAULT_DESCRIPTION), + ) + return heading + + def _generate_report(self): + rows = [] + sortedResult = self._sortResult(self.result) + for cid, (cls, cls_results) in enumerate(sortedResult): + # subtotal for a class + np = nf = ne = ns = 0 + for n, t, o, e in cls_results: + if n == 0: + np += 1 + elif n == 1: + nf += 1 + elif n == 2: + ne += 1 + else: + ns += 1 + + # format class description + if cls.mod == "__main__": + name = cls.name + else: + name = "%s" % (cls.name) + doc = cls.__doc__ and cls.__doc__.split("\n")[0] or "" + desc = doc and '%s: %s' % (name, doc) or name + + row = TemplateData.REPORT_CLASS_TMPL % dict( + style=(ne > 0 and 'errorClass' or nf > 0 + and 'failClass' or 'passClass'), + desc = desc, + count = np + nf + ne + ns, + Pass = np, + fail = nf, + error = ne, + skip = ns, + cid = 'c%s' % (cid + 1), + ) + rows.append(row) + + for tid, (n, t, o, e) in enumerate(cls_results): + self._generate_report_test(rows, cid, tid, n, t, o, e) + + report = TemplateData.REPORT_TMPL % dict( + test_list=''.join(rows), + count=str(self.success_count + self.failure_count + + self.error_count + self.skip_count), + Pass=str(self.success_count), + fail=str(self.failure_count), + error=str(self.error_count), + skip=str(self.skip_count), + ) + return report + + def _sortResult(self, result_list): + # unittest does not seems to run in any particular order. + # Here at least we want to group them together by class. + rmap = {} + classes = [] + for n, t, o, e in result_list: + if hasattr(t, '_tests'): + for inner_test in t._tests: + self._add_cls(rmap, classes, inner_test, + (n, inner_test, o, e)) + else: + self._add_cls(rmap, classes, t, (n, t, o, e)) + classort = lambda s: str(s) + sortedclasses = sorted(classes, key=classort) + r = [(cls, rmap[str(cls)]) for cls in sortedclasses] + return r + + def _add_cls(self, rmap, classes, test, data_tuple): + if hasattr(test, 'test'): + test = test.test + if test.__class__ == subunit.RemotedTestCase: + #print(test._RemotedTestCase__description.rsplit('.', 1)[0]) + cl = test._RemotedTestCase__description.rsplit('.', 1)[0] + mod = cl.rsplit('.', 1)[0] + cls = ClassInfoWrapper(cl, mod) + else: + cls = ClassInfoWrapper(str(test.__class__), str(test.__module__)) + if not str(cls) in rmap: + rmap[str(cls)] = [] + classes.append(cls) + rmap[str(cls)].append(data_tuple) + + def _generate_report_test(self, rows, cid, tid, n, t, o, e): + # e.g. 'pt1.1', 'ft1.1', etc + # ptx.x for passed/skipped tests and ftx.x for failed/errored tests. + has_output = bool(o or e) + tid = ((n == 0 or n == 3) and + 'p' or 'f') + 't%s.%s' % (cid + 1, tid + 1) + name = t.id().split('.')[-1] + # if shortDescription is not the function name, use it + if t.shortDescription().find(name) == -1: + doc = t.shortDescription() + else: + doc = None + desc = doc and ('%s: %s' % (name, doc)) or name + tmpl = (has_output and TemplateData.REPORT_TEST_WITH_OUTPUT_TMPL + or TemplateData.REPORT_TEST_NO_OUTPUT_TMPL) + + script = TemplateData.REPORT_TEST_OUTPUT_TMPL % dict( + id=tid, + output=saxutils.escape(o + e), + ) + + row = tmpl % dict( + tid=tid, + Class=((n == 0 or n == 3) and 'hiddenRow' or 'none'), + style=(n == 2 and 'errorCase' or + (n == 1 and 'failCase' or 'none')), + desc=desc, + script=script, + status=TemplateData.STATUS[n], + ) + rows.append(row) + if not has_output: + return + + def _generate_ending(self): + return TemplateData.ENDING_TMPL + + def startTestRun(self): + super(HtmlOutput, self).startTestRun() + + +class FileAccumulator(testtools.StreamResult): + + def __init__(self): + super(FileAccumulator, self).__init__() + self.route_codes = collections.defaultdict(io.BytesIO) + + def status(self, **kwargs): + if kwargs.get('file_name') != 'stdout': + return + file_bytes = kwargs.get('file_bytes') + if not file_bytes: + return + route_code = kwargs.get('route_code') + stream = self.route_codes[route_code] + stream.write(file_bytes) + + +def main(): + if len(sys.argv) < 2: + print("Need at least one argument: path to subunit log.") + exit(1) + subunit_file = sys.argv[1] + if len(sys.argv) > 2: + html_file = sys.argv[2] + else: + html_file = 'results.html' + + html_result = HtmlOutput(html_file) + stream = open(subunit_file, 'rb') + + # Feed the subunit stream through both a V1 and V2 parser. + # Depends on having the v2 capable libraries installed. + # First V2. + # Non-v2 content and captured non-test output will be presented as file + # segments called stdout. + suite = subunit.ByteStreamToStreamResult(stream, non_subunit_name='stdout') + # The HTML output code is in legacy mode. + result = testtools.StreamToExtendedDecorator(html_result) + # Divert non-test output + accumulator = FileAccumulator() + result = testtools.StreamResultRouter(result) + result.add_rule(accumulator, 'test_id', test_id=None) + result.startTestRun() + suite.run(result) + # Now reprocess any found stdout content as V1 subunit + for bytes_io in accumulator.route_codes.values(): + bytes_io.seek(0) + suite = subunit.ProtocolTestCase(bytes_io) + suite.run(html_result) + result.stopTestRun() + + +if __name__ == '__main__': + main() diff --git a/modules/openstack_project/files/slave_scripts/tardiff.py b/modules/openstack_project/files/slave_scripts/tardiff.py new file mode 100755 index 0000000000..8a66f6d21c --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/tardiff.py @@ -0,0 +1,189 @@ +#!/usr/bin/python + +# tardiff.py -- compare the tar package with git archive. Error out if +# it's different. The files to exclude are stored in a file, one per line, +# and it's passed as argument to this script. +# +# You should run this script from the project directory. For example, if +# you are verifying the package for glance project, you should run this +# script from that directory. + +import getopt +import sys +import os +import commands + + +class OpenStackTarDiff: + """ main class to verify tar generated in each openstack projects """ + + def __init__(self): + self.init_vars() + self.validate_args() + self.check_env() + + def check_env(self): + """ exit if dist/ directory already exists """ + if not self.package and os.path.exists(self.dist_dir): + self.error( + "dist directory '%s' exist. Please remove it before " + "running this script" % self.dist_dir) + + def validate_args(self): + try: + opts = getopt.getopt(sys.argv[1:], 'hvp:e:', + ['help', 'verbose', 'package=', + 'exclude='])[0] + except getopt.GetoptError: + self.usage('invalid option selected') + + for opt, value in opts: + if (opt in ('-h', '--help')): + self.usage() + elif (opt in ('-e', '--exclude')): + self.e_file = value + elif (opt in ('-p', '--package')): + self.package = value + elif (opt in ('-v', '--verbose')): + self.verbose = True + else: + self.usage('unknown option : ' + opt) + if not self.e_file: + self.usage('specify file name containing list of files to ' + 'exclude in tar diff') + if not os.path.exists(self.e_file): + self.usage("file '%s' does not exist" % self.e_file) + if self.package and not os.path.exists(self.package): + self.usage("package '%s' specified, but does not " + "exist" % self.package) + + def init_vars(self): + self.dist_dir = 'dist/' + self.verbose = False + + self.e_file = None + self.project_name = None + self.prefix = None + self.package = None + self.sdist_files = [] + self.exclude_files = [] + self.git_files = [] + self.missing_files = [] + + def verify(self): + self.get_exclude_files() + self.get_project_name() + self.get_sdist_files() + self.prefix = self.sdist_files[0] + self.get_git_files() + + for file in self.git_files: + if os.path.basename(file) in self.exclude_files: + self.debug("excluding file '%s'" % file) + continue + + if file not in self.sdist_files: + self.missing_files.append(file) + else: + #self.debug("file %s matches" % file) + pass + if len(self.missing_files) > 0: + self.error("files missing in package: %s" % self.missing_files) + print "SUCCESS: Generated package '%s' is valid" % self.package + + def get_project_name(self): + """ get git project name """ + self.project_name = os.path.basename(os.path.abspath(os.curdir)) + + def get_exclude_files(self): + """ read the file and get file list """ + fh = open(self.e_file, 'r') + content = fh.readlines() + fh.close() + self.debug("files to exclude: %s" % content) + + # remove trailing new lines. + self.exclude_files = [x.strip() for x in content] + + def get_git_files(self): + """ read file list from git archive """ + git_tar = os.path.join(os.getcwd(), '%s.tar' % self.project_name) + try: + a_cmd = ("git archive -o %s HEAD --prefix=%s" % + (git_tar, self.prefix)) + self.debug("executing command '%s'" % a_cmd) + (status, out) = commands.getstatusoutput(a_cmd) + if status != 0: + self.debug("command '%s' returned status '%s'" % + (a_cmd, status)) + if os.path.exists(git_tar): + os.unlink(git_tar) + self.error('git archive failed: %s' % out) + except Exception as err: + if os.path.exists(git_tar): + os.unlink(git_tar) + self.error('git archive failed: %s' % err) + + try: + tar_cmd = "tar tf %s" % git_tar + self.debug("executing command '%s'" % tar_cmd) + (status, out) = commands.getstatusoutput(tar_cmd) + if status != 0: + self.error('invalid tar file: %s' % git_tar) + self.git_files = out.split('\n') + self.debug("Removing git archive ... %s ..." % git_tar) + os.remove(git_tar) + except Exception as err: + self.error('unable to read tar: %s' % err) + + def get_sdist_files(self): + """ create package for project and get file list in it""" + if not self.package: + try: + sdist_cmd = "python setup.py sdist" + self.debug("executing command '%s'" % sdist_cmd) + (status, out) = commands.getstatusoutput(sdist_cmd) + if status != 0: + self.error("command '%s' failed" % sdist_cmd) + except Exception as err: + self.error("command '%s' failed" % (sdist_cmd, err)) + + self.package = os.listdir(self.dist_dir)[0] + self.package = os.path.join(self.dist_dir, self.package) + tar_cmd = "tar tzf %s" % self.package + try: + self.debug("executing command '%s'" % tar_cmd) + (status, out) = commands.getstatusoutput(tar_cmd) + if status != 0: + self.error("command '%s' failed" % tar_cmd) + #self.debug(out) + self.sdist_files = out.split('\n') + except Exception as err: + self.error("command '%s' failed: %s" % (tar_cmd, err)) + + def debug(self, msg): + if self.verbose: + sys.stdout.write('DEBUG: %s\n' % msg) + + def error(self, msg): + sys.stderr.write('ERROR: %s\n' % msg) + sys.exit(1) + + def usage(self, msg=None): + if msg: + stream = sys.stderr + else: + stream = sys.stdout + stream.write("usage: %s [--help|h] [-v] " + "[-p|--package=sdist_package.tar.gz] " + "-e|--exclude=filename\n" % os.path.basename(sys.argv[0])) + if msg: + stream.write("\nERROR: " + msg + "\n") + exitCode = 1 + else: + exitCode = 0 + sys.exit(exitCode) + +if __name__ == '__main__': + tardiff = OpenStackTarDiff() + tardiff.verify() diff --git a/modules/openstack_project/files/slave_scripts/update-pip-cache.sh b/modules/openstack_project/files/slave_scripts/update-pip-cache.sh new file mode 100755 index 0000000000..e1f08f0990 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/update-pip-cache.sh @@ -0,0 +1,10 @@ +#!/bin/bash -xe + +mkdir -p ~/cache/pip +VENV=`mktemp -d` +virtualenv --no-site-packages $VENV +cd $VENV +. bin/activate +PIP_DOWNLOAD_CACHE=~/cache/pip pip install `cat ~/devstack/files/pips/*` +cd +rm -fr $VENV diff --git a/modules/openstack_project/files/slave_scripts/upstream_translation_horizon.sh b/modules/openstack_project/files/slave_scripts/upstream_translation_horizon.sh new file mode 100755 index 0000000000..983594bb2a --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/upstream_translation_horizon.sh @@ -0,0 +1,59 @@ +#!/bin/bash -xe + +# Copyright 2014 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# The script is to push the updated English po to Transifex. + +PROJECT="horizon" + +if [ ! `echo $ZUUL_REFNAME | grep master` ] +then + exit 0 +fi + +git config user.name "OpenStack Jenkins" +git config user.email "jenkins@openstack.org" + +# Initialize the transifex client, if there's no .tx directory +if [ ! -d .tx ] ; then + tx init --host=https://www.transifex.com +fi +# Horizon JavaScript Translations +tx set --auto-local -r ${PROJECT}.${PROJECT}-js-translations \ +"${PROJECT}/locale//LC_MESSAGES/djangojs.po" --source-lang en \ +--source-file ${PROJECT}/locale/en/LC_MESSAGES/djangojs.po -t PO --execute +# Horizon Translations +tx set --auto-local -r ${PROJECT}.${PROJECT}-translations \ +"${PROJECT}/locale//LC_MESSAGES/django.po" --source-lang en \ +--source-file ${PROJECT}/locale/en/LC_MESSAGES/django.po -t PO --execute +# OpenStack Dashboard Translations +tx set --auto-local -r ${PROJECT}.openstack-dashboard-translations \ +"openstack_dashboard/locale//LC_MESSAGES/django.po" --source-lang en \ +--source-file openstack_dashboard/locale/en/LC_MESSAGES/django.po -t PO --execute + +# Invoke run_tests.sh to update the po files +# Or else, "../manage.py makemessages" can be used. +./run_tests.sh --makemessages -V + +# Add all changed files to git +git add ${PROJECT}/locale/en/LC_MESSAGES/* +git add openstack_dashboard/locale/en/LC_MESSAGES/* + +if [ `git diff --cached | egrep -v "(POT-Creation-Date|^[\+\-]#|^\+{3}|^\-{3})" | egrep -c "^[\-\+]"` -gt 0 ] +then + # Push source file changes to transifex + tx --debug --traceback push -s +fi + diff --git a/modules/openstack_project/files/slave_scripts/upstream_translation_update.sh b/modules/openstack_project/files/slave_scripts/upstream_translation_update.sh new file mode 100755 index 0000000000..e93d6b5d15 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/upstream_translation_update.sh @@ -0,0 +1,46 @@ +#!/bin/bash -xe + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +PROJECT=$1 + +if [ ! `echo $ZUUL_REFNAME | grep master` ] +then + exit 0 +fi + +git config user.name "OpenStack Jenkins" +git config user.email "jenkins@openstack.org" + +# Initialize the transifex client, if there's no .tx directory +if [ ! -d .tx ] ; then + tx init --host=https://www.transifex.com +fi +tx set --auto-local -r ${PROJECT}.${PROJECT}-translations "${PROJECT}/locale//LC_MESSAGES/${PROJECT}.po" --source-lang en --source-file ${PROJECT}/locale/${PROJECT}.pot -t PO --execute + +# Update the .pot file +python setup.py extract_messages +PO_FILES=`find ${PROJECT}/locale -name '*.po'` +if [ -n "$PO_FILES" ] +then + # Use updated .pot file to update translations + python setup.py update_catalog --no-fuzzy-matching --ignore-obsolete=true +fi +# Add all changed files to git +git add $PROJECT/locale/* + +if [ ! `git diff-index --quiet HEAD --` ] +then + # Push .pot changes to transifex + tx --debug --traceback push -s +fi diff --git a/modules/openstack_project/files/slave_scripts/upstream_translation_update_manuals.sh b/modules/openstack_project/files/slave_scripts/upstream_translation_update_manuals.sh new file mode 100755 index 0000000000..d3d5735bc4 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/upstream_translation_update_manuals.sh @@ -0,0 +1,70 @@ +#!/bin/bash -xe + +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# The script is to push the updated PoT to Transifex. + +PROJECT=$1 + +DocFolder="doc" +if [ $PROJECT = "api-site" ] ; then + DocFolder="./" +fi + +if [ ! `echo $ZUUL_REFNAME | grep master` ] +then + exit 0 +fi + +git config user.name "OpenStack Jenkins" +git config user.email "jenkins@openstack.org" + +# Initialize the transifex client, if there's no .tx directory +if [ ! -d .tx ] ; then + tx init --host=https://www.transifex.com +fi + +# Generate pot one by one +for FILE in ${DocFolder}/* +do + DOCNAME=${FILE#${DocFolder}/} + # high-availability-guide needs to create new DocBook files + if [ "$DOCNAME" == "high-availability-guide" ] + then + asciidoc -b docbook -d book -o - ${DocFolder}/high-availability-guide/ha-guide.txt \ +| xsltproc -o - /usr/share/xml/docbook/stylesheet/docbook5/db4-upgrade.xsl - \ +| xmllint --format - | sed -e 's, ${DocFolder}/high-availability-guide/bk-ha-guide.xml + fi + # Update the .pot file + ./tools/generatepot ${DOCNAME} + if [ -f ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot ] + then + # Add all changed files to git + git add ${DocFolder}/${DOCNAME}/locale/* + # Set auto-local + tx set --auto-local -r openstack-manuals-i18n.${DOCNAME} \ +"${DocFolder}/${DOCNAME}/locale/.po" --source-lang en \ +--source-file ${DocFolder}/${DOCNAME}/locale/${DOCNAME}.pot \ +-t PO --execute + fi +done + +if [ ! `git diff --cached --quiet HEAD --` ] +then + # Push .pot changes to transifex + tx --debug --traceback push -s +fi diff --git a/modules/openstack_project/files/slave_scripts/version-properties.sh b/modules/openstack_project/files/slave_scripts/version-properties.sh new file mode 100755 index 0000000000..7dffc05e9d --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/version-properties.sh @@ -0,0 +1,29 @@ +#!/bin/bash -ex +# +# This is a script that helps us version build artifacts. It retrieves +# git info and generates version strings. +# + +# get version info from scm +SCM_TAG=`git describe --abbrev=0 --tags` || true +SCM_SHA=`git rev-parse --short HEAD` || true + +# assumes format is like this '0.0.4-2-g135721c' +COMMITS_SINCE_TAG=`git describe | awk '{split($0,a,"-"); print a[2]}'` || true + +# just use git sha if there is no tag yet. +if [[ "${SCM_TAG}" == "" ]]; then + SCM_TAG=$SCM_SHA +fi + +# General build version should be something like '0.0.4.3.d4ee90c' +# Release build version should be something like '0.0.5' +if [[ "${COMMITS_SINCE_TAG}" == "" ]]; then + PROJECT_VER=$SCM_TAG +else + PROJECT_VER="$SCM_TAG.$COMMITS_SINCE_TAG.$SCM_SHA"; +fi + +echo "SCM_SHA=$SCM_SHA" >version.properties +echo "PROJECT_VER=$PROJECT_VER" >>version.properties +echo "COMMITS_SINCE_TAG=$COMMITS_SINCE_TAG" >>version.properties diff --git a/modules/openstack_project/files/slave_scripts/wait_for_nova.sh b/modules/openstack_project/files/slave_scripts/wait_for_nova.sh new file mode 100755 index 0000000000..564bb5859a --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/wait_for_nova.sh @@ -0,0 +1,12 @@ +#!/bin/bash -xe + +URL=$1 + +echo "Jenkins: Waiting for Nova to start on infrastructure node" +RET=7 +while [ $RET != 0 ]; do + curl -s $URL >/dev/null + RET=$? + sleep 1 +done +echo "Jenkins: Nova is running." diff --git a/modules/openstack_project/files/slave_scripts/wait_for_puppet.sh b/modules/openstack_project/files/slave_scripts/wait_for_puppet.sh new file mode 100755 index 0000000000..9812a15a98 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/wait_for_puppet.sh @@ -0,0 +1,21 @@ +#!/bin/bash -xe + +# wait_for_pupet.sh LOGFILE HOSTNAME [HOSTNAME...] +# Search LOGFILE for puppet completion on each host + +FINISH_RE="puppet-agent\[.*\]: Finished catalog run in .* seconds" +LOGFILE=$1 +shift +HOSTS=$@ + +echo "Jenkins: Waiting for puppet to complete on all nodes" +DONE=0 +while [ $DONE != 1 ]; do + DONE=1 + for hostname in $HOSTS + do + if !(grep "$hostname $FINISH_RE" $LOGFILE >/dev/null); then DONE=0; fi + done + sleep 5 +done +echo "Jenkins: Puppet is complete." diff --git a/modules/openstack_project/files/slave_scripts/zuul_swift_upload.py b/modules/openstack_project/files/slave_scripts/zuul_swift_upload.py new file mode 100755 index 0000000000..262c82c1f6 --- /dev/null +++ b/modules/openstack_project/files/slave_scripts/zuul_swift_upload.py @@ -0,0 +1,142 @@ +#!/usr/bin/python +# +# Copyright 2014 Rackspace Australia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utility to upload folders to swift using the form post middleware +credentials provided by zuul +""" + +import argparse +import magic +import os +import requests +import tempfile + + +def generate_log_index(file_list, logserver_prefix, swift_destination_prefix): + """Create an index of logfiles and links to them""" + + output = 'Index of results' + output += '
    ' + for f in file_list: + file_url = os.path.join(logserver_prefix, swift_destination_prefix, + f['filename']) + + output += '
  • ' + output += '%s' % (file_url, f['filename']) + output += '
    • ' + + output += '
' + output += '' + return output + + +def make_index_file(file_list, logserver_prefix, swift_destination_prefix, + index_filename='index.html'): + """Writes an index into a file for pushing""" + + index_content = generate_log_index(file_list, logserver_prefix, + swift_destination_prefix) + tempdir = tempfile.mkdtemp() + fd = open(os.path.join(tempdir, index_filename), 'w') + fd.write(index_content) + return os.path.join(tempdir, index_filename) + + +def swift_form_post_submit(file_list, url, hmac_body, signature): + """Send the files to swift via the FormPost middleware""" + + # We are uploading the file_list as an HTTP POST multipart encoded. + # First grab out the information we need to send back from the hmac_body + payload = {} + + (object_prefix, + payload['redirect'], + payload['max_file_size'], + payload['max_file_count'], + payload['expires']) = hmac_body.split('\\n') + payload['signature'] = signature + + if len(file_list) > payload['max_file_count']: + # We can't upload this many files! We'll do what we can but the job + # should be reconfigured + file_list = file_list[:payload['max_file_count']] + + files = {} + + for i, f in enumerate(file_list): + files['file%d' % (i + 1)] = (f['filename'], open(f['path'], 'rb'), + magic.from_file(f['path'], mime=True)) + + requests.post(url, data=payload, files=files) + + +def zuul_swift_upload(file_path, swift_url, swift_hmac_body, swift_signature, + logserver_prefix, swift_destination_prefix): + """Upload to swift using instructions from zuul""" + + # file_list: a list of dicts with {path=..., filename=...} where filename + # is appended to the end of the object (paths can be used) + file_list = [] + if os.path.isfile(file_path): + file_list.append({'filename': os.path.basename(file_path), + 'path': file_path}) + index_file = file_path + elif os.path.isdir(file_path): + for path, folders, files in os.walk(file_path): + for f in files: + full_path = os.path.join(path, f) + relative_name = os.path.relpath(full_path, file_path) + file_list.append({'filename': relative_name, + 'path': full_path}) + index_file = make_index_file(file_list, logserver_prefix, + swift_destination_prefix) + file_list.append({'filename': os.path.basename(index_file), + 'path': index_file}) + + swift_form_post_submit(file_list, swift_url, swift_hmac_body, + swift_signature) + + return (logserver_prefix + swift_destination_prefix + + os.path.basename(index_file)) + + +def grab_args(): + """Grab and return arguments""" + parser = argparse.ArgumentParser( + description="Upload results to swift using instructions from zuul" + ) + parser.add_argument('-n', '--name', default="logs", + help='The instruction-set to use') + parser.add_argument('files', nargs='+', help='the file(s) to upload') + + return parser.parse_args() + +if __name__ == '__main__': + args = grab_args() + for file_path in args.files: + try: + result_url = zuul_swift_upload( + file_path, + os.environ['SWIFT_%s_URL' % args.name], + os.environ['SWIFT_%s_HMAC_BODY' % args.name], + os.environ['SWIFT_%s_SIGNATURE' % args.name], + os.environ['SWIFT_%s_LOGSERVER_PREFIX' % args.name], + os.environ['SWIFT_%s_DESTINATION_PREFIX' % args.name] + ) + print result_url + except KeyError as e: + print 'Environment variable %s not found' % e