Imported CI Scripts

2011-07-14 14:09:50 -04:00
commit f0ef49a2fd
10 changed files with 364 additions and 0 deletions
--- a/.config/tarmac/tarmac.conf
+++ b/.config/tarmac/tarmac.conf
@@ -0,0 +1,32 @@
+[Tarmac]
+rejected_branch_status = Work in progress
+
+[lp:nova]
+verify_command=/var/lib/jenkins/test_nova.sh
+
+[lp:~hudson-openstack/nova/milestone-proposed]
+verify_command=/var/lib/jenkins/test_nova.sh
+
+[lp:openstack-dashboard]
+verify_command=bash run_tests.sh
+
+[lp:glance]
+verify_command=python setup.py test
+
+[lp:~hudson-openstack/glance/milestone-proposed]
+verify_command=python setup.py test
+
+[lp:swift]
+verify_command=python setup.py test
+
+[lp:swift/1.1]
+verify_command=python setup.py test
+
+[lp:swift/1.2]
+verify_command=python setup.py test
+
+[lp:~hudson-openstack/swift/milestone-proposed]
+verify_command=python setup.py test
+
+[lp:burrow]
+verify_command=python setup.py test
--- a/bin_ppa_script.sh
+++ b/bin_ppa_script.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+
+set -e
+
+if [ -z "$PROJECT" ]
+then
+	echo '$PROJECT not set.'
+	exit 1
+fi
+
+HUDSON=http://localhost:8080/
+PKGRECORDFILE=$HOME/binpkgversions
+JENKINS_TARBALL_JOB=${JENKINS_TARBALL_JOB:-$PROJECT-tarball}
+BZR_BRANCH=${BZR_BRANCH:-lp:~openstack-ubuntu-packagers/$PROJECT/ubuntu}
+PPAS=${PPAS:-ppa:$PROJECT-core/trunk}
+PACKAGING_REVNO=${PACKAGING_REVNO:--1}
+
+# Clean up after previous build
+rm -rf build dist.zip
+mkdir build
+
+# Grab the most recently built artifacts
+wget $HUDSON/job/${JENKINS_TARBALL_JOB}/lastBuild/artifact/dist/*zip*/dist.zip
+
+# Shove them in build/
+unzip dist.zip -d build
+
+cd build
+
+tarball="$(echo dist/$PROJECT*.tar.gz)"
+version="${tarball%.tar.gz}"
+version="${version#*$PROJECT-}"
+if [ -n "${EXTRAVERSION}" ]
+then
+    version="${version%~*}${EXTRAVERSION}~${version#*~}"
+fi
+tar xvzf "${tarball}"
+echo ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz"
+ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz"
+
+# Overlay packaging
+# (Intentionally using the natty branch. For these PPA builds, we don't need to diverge
+# (yet, at least), so it makes the branch management easier this way.
+# Note: Doing a checkout and deleting .bzr afterwards instead of just doing an export,
+# because export refuses to overlay over an existing directory, so this was easier.
+# (We need to not have the .bzr in there, otherwise vcsversion.py might get overwritten)
+bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-*
+cd $PROJECT-*
+PACKAGING_REVNO="$(bzr revno --tree)"
+rm -rf .bzr
+
+# Please don't change this. It's the only way I'll get notified
+# if an upload fails.
+export DEBFULLNAME="Soren Hansen"
+export DEBEMAIL="soren@openstack.org"
+
+buildno=1
+while true
+do
+	pkgversion="${version}-0ubuntu0ppa1~${buildno}"
+	if grep "$PROJECT $pkgversion" "$PKGRECORDFILE"
+	then
+		echo "We've already built a $pkgversion of $PROJECT. Incrementing build number."
+		buildno=$(($buildno + 1))
+	else
+		echo "$PROJECT $pkgversion" >> "$PKGRECORDFILE"
+		break
+	fi
+done
+# Doing this in here so that we have buildno
+server_name=${PACKAGE}-`echo ${pkgversion} | sed 's/\~//g'`
+echo "Launching a Cloud Server"
+python ${HOME}/launch_node.py ${server_name}
+cp node.sh ..
+dch -b --force-distribution --v "${pkgversion}" "Automated PPA build. Packaging revision: ${PACKAGING_REVNO}." -D maverick
+dpkg-buildpackage -rfakeroot -sa -k32EE128C
+cd ..
--- a/dashboard_test.sh
+++ b/dashboard_test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+bzr branch lp:django-nova
+python tools/install_venv.py django-nova
+
+cp local/local_settings.py.example local/local_settings.py
+tools/with_venv.sh dashboard/manage.py  test
--- a/launch_node.py
+++ b/launch_node.py
@@ -0,0 +1,45 @@
+from libcloud.types import Provider
+from libcloud.providers import get_driver
+from libcloud.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
+import os, sys
+
+CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME']
+CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY']
+try:
+  node_name = sys.argv[1]
+except:
+  print "Node Name required!"
+  sys.exit(1)
+
+Driver = get_driver(Provider.RACKSPACE)
+conn = Driver(CLOUD_SERVERS_USERNAME, CLOUD_SERVERS_API_KEY)
+
+# read your public key in
+sd = SSHKeyDeployment(open(os.path.expanduser("~/.ssh/id_rsa.pub")).read())
+# a simple script to install puppet post boot, can be much more complicated.
+script = ScriptDeployment("""
+perl -ple 's,main,main universe,' -i /etc/apt/sources.list
+apt-get update
+apt-get install -y --force-yes git rubygems
+gem install puppet
+git clone git://github.com/openstack/openstack-ci-puppet.git
+cd openstack-ci-puppet
+puppet apply --modulepath=`pwd`/modules manifests/server.pp
+""")
+
+# a task that first installs the ssh key, and then runs the script
+msd = MultiStepDeployment([sd, script])
+
+
+images = conn.list_images()
+
+size = [sz for sz in conn.list_sizes() if sz.id == '3'][0]
+image = [img for img in conn.list_images() if img.id == '76'][0]
+
+
+# deploy_node takes the same base keyword arguments as create_node.
+node = conn.deploy_node(name=node_name, image=image, size=size, deploy=msd)
+
+with open("%s.node.sh" % node_name,"w") as node_file:
+  node_file.write("ipAddr=%s\n" % node.public_ip[0])
+  node_file.write("nodeId=%s\n" % node.id)
--- a/launch_slave.py
+++ b/launch_slave.py
@@ -0,0 +1,45 @@
+from libcloud.types import Provider
+from libcloud.providers import get_driver
+from libcloud.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
+import os, sys
+
+CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME']
+CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY']
+try:
+  node_name = sys.argv[1]
+except:
+  print "Node Name required!"
+  sys.exit(1)
+
+Driver = get_driver(Provider.RACKSPACE)
+conn = Driver(CLOUD_SERVERS_USERNAME, CLOUD_SERVERS_API_KEY)
+
+# read your public key in
+sd = SSHKeyDeployment(open(os.path.expanduser("~/.ssh/id_rsa.pub")).read())
+# a simple script to install puppet post boot, can be much more complicated.
+script = ScriptDeployment("""
+perl -ple 's,main,main universe,' -i /etc/apt/sources.list
+apt-get update
+apt-get install -y --force-yes git rubygems
+gem install puppet
+git clone git://github.com/openstack/openstack-ci-puppet.git
+cd openstack-ci-puppet
+puppet apply --modulepath=`pwd`/modules manifests/slave.pp
+""")
+
+# a task that first installs the ssh key, and then runs the script
+msd = MultiStepDeployment([sd, script])
+
+
+images = conn.list_images()
+
+size = [sz for sz in conn.list_sizes() if sz.id == '3'][0]
+image = [img for img in conn.list_images() if img.id == '76'][0]
+
+
+# deploy_node takes the same base keyword arguments as create_node.
+node = conn.deploy_node(name=node_name, image=image, size=size, deploy=msd)
+
+with open("%s.node.sh" % node_name,"w") as node_file:
+  node_file.write("ipAddr=%s\n" % node.public_ip[0])
+  node_file.write("nodeId=%s\n" % node.id)
--- a/2
+++ b/2
@@ -0,0 +1,2 @@
+nova: add-apt-repository ppa:nova-core/trunk ; apt-get update ; apt-get
+build-dep nova ; easy_install nosexcover 
--- a/ppa_script.sh
+++ b/ppa_script.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+
+set -e
+
+if [ -z "$PROJECT" ]
+then
+	echo '$PROJECT not set.'
+	exit 1
+fi
+
+HUDSON=http://localhost:8080/
+PKGRECORDFILE=$HOME/pkgversions
+BZR_BRANCH=${BZR_BRANCH:-lp:~openstack-ubuntu-packagers/$PROJECT/ubuntu}
+PPAS=${PPAS:-ppa:$PROJECT-core/trunk}
+PACKAGING_REVNO=${PACKAGING_REVNO:--1}
+series=${series:-lucid}
+
+cd build
+
+tarball="$(echo dist/$PROJECT*.tar.gz)"
+version="${tarball%.tar.gz}"
+version="${version#*$PROJECT-}"
+base_version=$version
+if [ -n "${EXTRAVERSION}" ]
+then
+    version="${version%~*}${EXTRAVERSION}~${version#*~}"
+fi
+tar xvzf "${tarball}"
+echo ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz"
+ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz"
+
+# Overlay packaging
+# (Intentionally using the natty branch. For these PPA builds, we don't need to diverge
+# (yet, at least), so it makes the branch management easier this way.
+# Note: Doing a checkout and deleting .bzr afterwards instead of just doing an export,
+# because export refuses to overlay over an existing directory, so this was easier.
+# (We need to not have the .bzr in there, otherwise vcsversion.py might get overwritten)
+echo bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-*
+bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-*
+cd $PROJECT-*
+PACKAGING_REVNO="$(bzr revno --tree)"
+rm -rf .bzr
+
+# Please don't change this. It's the only way I'll get notified
+# if an upload fails.
+export DEBFULLNAME="Soren Hansen"
+export DEBEMAIL="soren@openstack.org"
+
+buildno=1
+while true
+do
+	pkgversion="${version}-0ubuntu0ppa1~${series}${buildno}"
+	if grep "$PROJECT $pkgversion" "$PKGRECORDFILE"
+	then
+		echo "We've already built a $pkgversion of $PROJECT. Incrementing build number."
+		buildno=$(($buildno + 1))
+	else
+		echo "$PROJECT $pkgversion" >> "$PKGRECORDFILE"
+		break
+	fi
+done
+dch -b --force-distribution --v "${pkgversion}" "Automated PPA build. Packaging revision: ${PACKAGING_REVNO}." -D $series
+dpkg-buildpackage -rfakeroot -S -sa -k32EE128C
+if ! [ "$DO_UPLOAD" = "no" ]
+then
+	for ppa in $PPAS
+	do
+		dput --force $ppa "../${PROJECT}_${pkgversion}_source.changes"
+	done
+fi
+cd ..
--- a/tarball_bzr_diff.sh
+++ b/tarball_bzr_diff.sh
@@ -0,0 +1,57 @@
+#!/bin/sh
+
+retval=0
+
+STATEPATH=${JENKINS_HOME:-$HOME}
+BNT=in_bzr_but_not_in_tarball.txt
+TNB=in_tarball_but_not_in_bzr.txt
+BNTSAVED=$STATEPATH/$BNT.saved
+TNBSAVED=$STATEPATH/$TNB.saved
+
+bzr ls -R . --versioned | sort > bzr.lst
+tar tzf nova-*.tar.gz  | cut -f2- -d/ | grep -v ^$ | sort -g > tarball.lst
+rm -rf dist dist.zip
+diff -u bzr.lst tarball.lst | grep -v ^--- | grep -v ^+++  > diff
+grep ^- diff | sed -e s/^.// > $BNT
+grep ^+ diff | sed -e s/^.// > $TNB
+
+if [ "$1" = "ack" ]
+then
+        cp $BNT $BNTSAVED
+        cp $TNB $TNBSAVED
+        exit 0
+fi
+
+> report.txt
+
+if ! diff -Nq $BNTSAVED $BNT > /dev/null
+then
+        retval=1
+        echo "The list of files in bzr, but not in the tarball changed." >> report.txt
+        echo "Lines beginning with - denote files that were either removed from bzr or recently included in the tarball." >> report.txt
+        echo "Lines beginning with + denote files that were either got added to bzr recently or got removed from the tarball." >> report.txt
+        diff -uN $BNTSAVED $BNT >> report.txt
+fi
+if ! diff -qN $TNBSAVED $TNB > /dev/null
+then
+        retval=1
+        echo "The list of files in the tarball, but not in bzr changed." >> report.txt
+        echo "Lines beginning with - denote files that were removed from the tarball, but is still in bzr." >> report.txt
+        echo "Lines beginning with + denote files that were either got added to the tarball recently or which disappeared from bzr, but stayed in the tarball." >> report.txt
+	diff -uN $TNBSAVED $TNB >> report.txt
+fi
+
+mkdir -p html/
+
+echo '<html><title>Tarball vs bzr delta changes</title><body><pre>' > html/report.html
+cat report.txt >> html/report.html
+echo '</pre>' >> html/report.html
+
+if [ $retval = 1 ]
+then
+	echo "<p>If these differences are ok, <a href="http://hudson.openstack.org/job/nova-tarball-bzr-delta/build">run the job again</a> and check the 'ack' box.</p>" >> report.txt
+fi
+
+echo '</body></html>' >> html/report.html
+
+exit $retval
--- a/tarball_script.sh
+++ b/tarball_script.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -e
+
+if [ -z "$PROJECT" ]
+then
+	echo '$PROJECT not set.'
+	exit 1
+fi
+
+RECORDFILE=$HOME/tarballversions
+SEPARATOR=${SEPARATOR:-'~'}
+revno=$(bzr revno)
+datestamp="$(date +%Y%m%d)"
+
+if grep "^$PROJECT $revno$" "$RECORDFILE";
+then
+	echo "Tarball already built. Not rebuilding."
+	exit 0
+fi
+echo "$PROJECT $revno" '>>' "$RECORDFILE"
+
+python setup.py sdist
+tarball=$(echo dist/*.tar.gz)
+mv "$tarball" "dist/$(basename $tarball .tar.gz)${SEPARATOR}bzr${revno}.tar.gz"
--- a/test_nova.sh
+++ b/test_nova.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+bash run_tests.sh -N && python setup.py sdist