From f32f06b50b4865bbcd87d28c7a6be4b1c17c9f2e Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 22 May 2012 01:36:08 +0000 Subject: [PATCH] Make devstack nodes jenkins slaves. Add them as nodes with the tag "devstack-IMAGE" where IMAGE is, eg, oneiric, with a single executor. Change the reap job to delete these jenkins nodes. Change the delete script to simply mark them for deletion. It should then be called by a new job that is a post-build step for the devstack gate job. Another new job should be called as the first build step in the devstack gate, and it should invoke devstack-vm-inprogress.py to disable the node that the job just started running on. In this manner, we end up with single-use jenkins slaves. Change the gate script to expect to be run the host itself, so it no longer needs to ssh and scp/rsync files around. Add a new script, devstack-vm-gate-wrap.sh, which assists running the gate job on a separate devstack host, as is currently done, to test it out without requiring the full Jenkins infrastructure. Change-Id: I28902918406670163d32ae7c2a644055233dc1fa --- devstack-vm-delete.py | 13 +-- devstack-vm-gate-dev.sh | 27 ++++++ devstack-vm-gate-host.sh | 121 -------------------------- devstack-vm-gate-wrap.sh | 147 +++++++++++++++++++++++++++++++ devstack-vm-gate.sh | 167 +++++++++--------------------------- devstack-vm-inprogress.py | 58 +++++++++++++ devstack-vm-launch.py | 31 ++++++- devstack-vm-reap.py | 29 +++++-- devstack-vm-update-image.py | 4 +- myjenkins.py | 93 ++++++++++++++++++++ vmdatabase.py | 18 ++-- 11 files changed, 435 insertions(+), 273 deletions(-) create mode 100755 devstack-vm-gate-dev.sh delete mode 100755 devstack-vm-gate-host.sh create mode 100755 devstack-vm-gate-wrap.sh create mode 100755 devstack-vm-inprogress.py create mode 100644 myjenkins.py diff --git a/devstack-vm-delete.py b/devstack-vm-delete.py index 1707c8cc..ea694fec 100755 --- a/devstack-vm-delete.py +++ b/devstack-vm-delete.py @@ -26,20 +26,13 @@ import time import vmdatabase import utils -NODE_ID = sys.argv[1] - +NODE_NAME = sys.argv[1] def main(): db = vmdatabase.VMDatabase() - machine = db.getMachine(NODE_ID) - provider = machine.base_image.provider - - client = utils.get_client(provider) - - server = client.servers.get(machine.external_id) - utils.delete_server(server) - machine.delete() + machine = db.getMachineByJenkinsName(NODE_NAME) + machine.state = vmdatabase.DELETE if __name__ == '__main__': main() diff --git a/devstack-vm-gate-dev.sh b/devstack-vm-gate-dev.sh new file mode 100755 index 00000000..3901dc2e --- /dev/null +++ b/devstack-vm-gate-dev.sh @@ -0,0 +1,27 @@ +#!/bin/bash -x + +# Simulate what Jenkins does with the devstack-gate script. + +NODE_IP_ADDR=$1 + +cat >$WORKSPACE/test-env.sh <localrc -ACTIVE_TIMEOUT=60 -BOOT_TIMEOUT=90 -ASSOCIATE_TIMEOUT=60 -MYSQL_PASSWORD=secret -RABBIT_PASSWORD=secret -ADMIN_PASSWORD=secret -SERVICE_PASSWORD=secret -SERVICE_TOKEN=111222333444 -ROOTSLEEP=0 -ENABLED_SERVICES=$ENABLED_SERVICES -SKIP_EXERCISES=boot_from_volume,client-env,swift -SERVICE_HOST=127.0.0.1 -SYSLOG=True -SCREEN_LOGDIR=/opt/stack/screen-logs -FIXED_RANGE=10.1.0.0/24 -FIXED_NETWORK_SIZE=256 -EOF - -if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then - # We need to disable ratelimiting when running - # Tempest tests since so many requests are executed - echo "API_RATE_LIMIT=False" >> localrc - # Volume tests in Tempest require a number of volumes - # to be created, each of 1G size. Devstack's default - # volume backing file size is 2G, so we increase to 4G - echo "VOLUME_BACKING_FILE_SIZE=4G" >> localrc -fi - -# The vm template update job should cache some images in ~/files. -# Move them to where devstack expects: -if ls ~/cache/files/*; then - mv ~/cache/files/* /opt/stack/devstack/files -fi - -# Move the PIP cache into position: -sudo mkdir -p /var/cache/pip -sudo mv ~/cache/pip/* /var/cache/pip - -# Start with a fresh syslog -sudo stop rsyslog -sudo mv /var/log/syslog /var/log/syslog-pre-devstack -sudo touch /var/log/syslog -sudo chown /var/log/syslog --ref /var/log/syslog-pre-devstack -sudo chmod /var/log/syslog --ref /var/log/syslog-pre-devstack -sudo chmod a+r /var/log/syslog -sudo start rsyslog - -./stack.sh -if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then - ./tools/configure_tempest.sh - cd /opt/stack/tempest - nosetests --with-xunit -sv $DEVSTACK_GATE_TEMPEST_TESTS -else - ./exercise.sh -fi diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh new file mode 100755 index 00000000..bd3f2a63 --- /dev/null +++ b/devstack-vm-gate-wrap.sh @@ -0,0 +1,147 @@ +#!/bin/bash -x + +# Gate commits to several projects on a VM running those projects +# configured by devstack. + +# Copyright (C) 2011-2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +PROJECTS="openstack-dev/devstack openstack/nova openstack/glance openstack/keystone openstack/python-novaclient openstack/python-keystoneclient openstack/python-quantumclient openstack/python-glanceclient openstack/horizon openstack/tempest" + +# Set to 1 to run the Tempest test suite +export DEVSTACK_GATE_TEMPEST=${DEVSTACK_GATE_TEMPEST:-0} + +# Supply specific tests to Tempest in second argument +# For example, to execute only the server actions test, +# you would supply tempest.test.test_server_actions +export DEVSTACK_GATE_TEMPEST_TESTS=${DEVSTACK_GATE_TEMPEST_TESTS:-tempest} + +# Set this variable to skip updating the devstack-gate project itself. +# Useful in development so you can edit scripts in place and run them +# directly. Do not set in production. +# Normally not set, and we do include devstack-gate with the rest of +# the projects. +if [ -z "$SKIP_DEVSTACK_GATE_PROJECT" ]; then + PROJECTS="openstack-ci/devstack-gate $PROJECTS" +fi + +# Set this variable to include tempest in the test run. +if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then + PROJECTS="openstack/tempest $PROJECTS" +fi + +cd $WORKSPACE + +for PROJECT in $PROJECTS +do + echo "Setting up $PROJECT" + SHORT_PROJECT=`basename $PROJECT` + if [[ ! -e $SHORT_PROJECT ]]; then + echo " Need to clone" + git clone https://review.openstack.org/p/$PROJECT + fi + cd $SHORT_PROJECT + + BRANCH=$GERRIT_BRANCH + + # See if this project has this branch, if not, use master + git remote update + # Ensure that we don't have stale remotes around + git remote prune origin + if ! git branch -a |grep remotes/origin/$GERRIT_BRANCH>/dev/null; then + BRANCH=master + fi + git reset --hard + git clean -x -f -d -q + git checkout $BRANCH + git reset --hard remotes/origin/$BRANCH + git clean -x -f -d -q + + if [[ $GERRIT_PROJECT == $PROJECT ]]; then + echo " Merging proposed change" + git fetch https://review.openstack.org/p/$PROJECT $GERRIT_REFSPEC + git merge FETCH_HEAD + else + echo " Updating from origin" + git pull --ff-only origin $BRANCH + fi + cd $WORKSPACE +done + +# Set GATE_SCRIPT_DIR to point to devstack-gate in the workspace so that +# we are testing the proposed change from this point forward. +GATE_SCRIPT_DIR=$WORKSPACE/devstack-gate + +# Also, if we're testing devstack-gate, re-exec this script once so +# that we can test the new version of it. +if [[ $GERRIT_PROJECT == "openstack-ci/devstack-gate" ]] && [[ $RE_EXEC != "true" ]]; then + export RE_EXEC="true" + exec $GATE_SCRIPT_DIR/devstack-vm-gate-wrap.sh +fi + +# Make sure headers for the currently running kernel are installed: +sudo apt-get install -y --force-yes linux-headers-`uname -r` + +# Hpcloud provides no swap, but does have a partition mounted at /mnt +# we can use: +if [ `cat /proc/meminfo | grep SwapTotal | awk '{ print $2; }'` -eq 0 ] && + [ -b /dev/vdb ]; then + sudo umount /dev/vdb + sudo mkswap /dev/vdb + sudo swapon /dev/vdb +fi + +# The vm template update job should cache some images in ~/files. +# Move them to where devstack expects: +if ls ~/cache/files/*; then + mv ~/cache/files/* $WORKSPACE/devstack/files +fi + +# Move the PIP cache into position: +sudo mkdir -p /var/cache/pip +sudo mv ~/cache/pip/* /var/cache/pip + +# Start with a fresh syslog +sudo stop rsyslog +sudo mv /var/log/syslog /var/log/syslog-pre-devstack +sudo touch /var/log/syslog +sudo chown /var/log/syslog --ref /var/log/syslog-pre-devstack +sudo chmod /var/log/syslog --ref /var/log/syslog-pre-devstack +sudo chmod a+r /var/log/syslog +sudo start rsyslog + +# Run the test +$GATE_SCRIPT_DIR/devstack-vm-gate.sh +RETVAL=$? + +cd $WORKSPACE +# No matter what, archive logs +mkdir -p logs +rm -f logs/* + +sudo cp /var/log/syslog $WORKSPACE/logs/syslog.txt +cp $WORKSPACE/screen-logs/* $WORKSPACE/logs/ + +# Make sure jenkins can read all the logs +sudo chown -R jenkins.jenkins $WORKSPACE/logs/ +sudo chmod a+r $WORKSPACE/logs/ + +rename 's/\.log$/.txt/' $WORKSPACE/logs/* + +# Remove duplicate logs +rm $WORKSPACE/logs/*.*.txt + +exit $RETVAL diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 1d765f8d..553e071e 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -1,7 +1,7 @@ #!/bin/bash -x -# Gate commits to several projects on a VM running those projects -# configured by devstack. +# Script that is run on the devstack vm; configures and +# invokes devstack. # Copyright (C) 2011-2012 OpenStack LLC. # @@ -19,141 +19,52 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECTS="openstack-dev/devstack openstack/nova openstack/glance openstack/keystone openstack/python-novaclient openstack/python-keystoneclient openstack/python-quantumclient openstack/python-glanceclient openstack/horizon openstack/tempest" +set -o errexit -# Set to 1 to run the Tempest test suite -DEVSTACK_GATE_TEMPEST=${DEVSTACK_GATE_TEMPEST:-0} +export DEST=$WORKSPACE -# Supply specific tests to Tempest in second argument -# For example, to execute only the server actions test, -# you would supply tempest.test.test_server_actions -DEVSTACK_GATE_TEMPEST_TESTS=${DEVSTACK_GATE_TEMPEST_TESTS:-tempest} +cd $DEST/devstack -# Set this variable to skip updating the devstack-gate project itself. -# Useful in development so you can edit scripts in place and run them -# directly. Do not set in production. -# Normally not set, and we do include devstack-gate with the rest of -# the projects. -if [ -z "$SKIP_DEVSTACK_GATE_PROJECT" ]; then - PROJECTS="openstack-ci/devstack-gate $PROJECTS" -fi +ENABLED_SERVICES=g-api,g-reg,key,n-api,n-crt,n-obj,n-cpu,n-net,n-vol,n-sch,horizon,mysql,rabbit -# Set this variable to include tempest in the test run. if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then - PROJECTS="openstack/tempest $PROJECTS" + ENABLED_SERVICES=$ENABLED_SERVICES,tempest fi -# Set this to 1 to always keep the host around -ALWAYS_KEEP=${ALWAYS_KEEP:-0} +cat <localrc +ACTIVE_TIMEOUT=60 +BOOT_TIMEOUT=90 +ASSOCIATE_TIMEOUT=60 +MYSQL_PASSWORD=secret +RABBIT_PASSWORD=secret +ADMIN_PASSWORD=secret +SERVICE_PASSWORD=secret +SERVICE_TOKEN=111222333444 +SWIFT_HASH=1234123412341234 +ROOTSLEEP=0 +ENABLED_SERVICES=$ENABLED_SERVICES +SKIP_EXERCISES=boot_from_volume,client-env,swift +SERVICE_HOST=127.0.0.1 +SYSLOG=True +SCREEN_LOGDIR=$WORKSPACE/screen-logs +FIXED_RANGE=10.1.0.0/24 +FIXED_NETWORK_SIZE=256 +EOF -cd $WORKSPACE -mkdir -p logs -rm -f logs/* - -for PROJECT in $PROJECTS -do - echo "Setting up $PROJECT" - SHORT_PROJECT=`basename $PROJECT` - if [[ ! -e $SHORT_PROJECT ]]; then - echo " Need to clone" - git clone https://review.openstack.org/p/$PROJECT - fi - cd $SHORT_PROJECT - - BRANCH=$GERRIT_BRANCH - - # See if this project has this branch, if not, use master - git remote update - # Ensure that we don't have stale remotes around - git remote prune origin - if ! git branch -a |grep remotes/origin/$GERRIT_BRANCH>/dev/null; then - BRANCH=master - fi - git reset --hard - git clean -x -f -d -q - git checkout $BRANCH - git reset --hard remotes/origin/$BRANCH - git clean -x -f -d -q - - if [[ $GERRIT_PROJECT == $PROJECT ]]; then - echo " Merging proposed change" - git fetch https://review.openstack.org/p/$PROJECT $GERRIT_REFSPEC - git merge FETCH_HEAD - else - echo " Updating from origin" - git pull --ff-only origin $BRANCH - fi - cd $WORKSPACE -done - -# Set GATE_SCRIPT_DIR to point to devstack-gate in the workspace so that -# we are testing the proposed change from this point forward. -GATE_SCRIPT_DIR=$WORKSPACE/devstack-gate - -# Also, if we're testing devstack-gate, re-exec this script once so -# that we can test the new version of it. -if [[ $GERRIT_PROJECT == "openstack-ci/devstack-gate" ]] && [[ $RE_EXEC != "true" ]]; then - export RE_EXEC="true" - exec $GATE_SCRIPT_DIR/devstack-vm-gate.sh -fi - -$GATE_SCRIPT_DIR/devstack-vm-fetch.py oneiric > node_info.sh || exit $? -. node_info.sh - -scp -C $GATE_SCRIPT_DIR/devstack-vm-gate-host.sh $NODE_IP_ADDR: -RETVAL=$? -if [ $RETVAL != 0 ]; then - echo "Recording node run as failure." - if [ -n "$RESULT_ID" ]; then - $GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure - fi - echo "Deleting host" - $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID - exit $RETVAL -fi - -rsync -az --delete $WORKSPACE/ $NODE_IP_ADDR:workspace/ -RETVAL=$? -if [ $RETVAL != 0 ]; then - echo "Recording node run as failure." - if [ -n "$RESULT_ID" ]; then - $GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure - fi - echo "Deleting host" - $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID - exit $RETVAL -fi - -ssh $NODE_IP_ADDR ./devstack-vm-gate-host.sh $DEVSTACK_GATE_TEMPEST $DEVSTACK_GATE_TEMPEST_TESTS -RETVAL=$? -# No matter what, archive logs -scp -C -q $NODE_IP_ADDR:/var/log/syslog $WORKSPACE/logs/syslog.txt -scp -C -q $NODE_IP_ADDR:/opt/stack/screen-logs/* $WORKSPACE/logs/ -rename 's/\.log$/.txt/' $WORKSPACE/logs/* -# Remove duplicate logs -rm $WORKSPACE/logs/*.*.txt -# Copy XUnit test results from tempest, if run. if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then - scp -C -q $NODE_IP_ADDR:/opt/stack/tempest/nosetests.xml $WORKSPACE/tempest/ + # We need to disable ratelimiting when running + # Tempest tests since so many requests are executed + echo "API_RATE_LIMIT=False" >> localrc + # Volume tests in Tempest require a number of volumes + # to be created, each of 1G size. Devstack's default + # volume backing file size is 2G, so we increase to 4G + echo "VOLUME_BACKING_FILE_SIZE=4G" >> localrc fi -# Now check whether the run was a success -if [ -n "$RESULT_ID" ]; then - if [ $RETVAL = 0 ]; then - echo "Recording node run as success." - $GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID success - else - echo "Recording node run as failure." - $GATE_SCRIPT_DIR/devstack-vm-result.py $RESULT_ID failure - fi -fi - -if [ $RETVAL = 0 ] && [ $ALWAYS_KEEP = 0 ]; then - echo "Deleting host" - $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID - exit $RETVAL -else - #echo "Giving host to developer" - #$GATE_SCRIPT_DIR/devstack-vm-give.py $NODE_ID - exit $RETVAL +./stack.sh +./exercise.sh +if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then + ./tools/configure_tempest.sh + cd $DEST/tempest + nosetests --with-xunit -sv $DEVSTACK_GATE_TEMPEST_TESTS fi diff --git a/devstack-vm-inprogress.py b/devstack-vm-inprogress.py new file mode 100755 index 00000000..03de1bd3 --- /dev/null +++ b/devstack-vm-inprogress.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# Remove old devstack VMs that have been given to developers. + +# Copyright (C) 2011-2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import time +import getopt +import traceback +import ConfigParser + +import myjenkins +import vmdatabase +import utils +import novaclient + +NODE_NAME = sys.argv[1] +DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', + os.path.expanduser('~/devstack-gate-secure.conf')) + + +def main(): + db = vmdatabase.VMDatabase() + + config=ConfigParser.ConfigParser() + config.read(DEVSTACK_GATE_SECURE_CONFIG) + + jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), + config.get('jenkins', 'user'), + config.get('jenkins', 'apikey')) + jenkins.get_info() + + machine = db.getMachineByJenkinsName(NODE_NAME) + machine.state = vmdatabase.USED + + if machine.jenkins_name: + if jenkins.node_exists(machine.jenkins_name): + jenkins.disable_node(machine.jenkins_name, "Devstack build started") + + +if __name__ == '__main__': + main() diff --git a/devstack-vm-launch.py b/devstack-vm-launch.py index a2b56dcf..a0783606 100755 --- a/devstack-vm-launch.py +++ b/devstack-vm-launch.py @@ -25,12 +25,16 @@ import getopt import time import paramiko import traceback +import ConfigParser +import myjenkins import vmdatabase import utils PROVIDER_NAME = sys.argv[1] DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '') +DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', + os.path.expanduser('~/devstack-gate-secure.conf')) ABANDON_TIMEOUT = 900 # assume a machine will never boot if it hasn't # after this amount of time @@ -76,8 +80,21 @@ def launch_node(client, snap_image, image, flavor, last_name): print return server, machine +def create_jenkins_node(jenkins, machine): + name = '%sdevstack-%s-%s-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name, + machine.base_image.provider.name, machine.id) + machine.jenkins_name = name -def check_machine(client, machine, error_counts): + jenkins.create_node(name, numExecutors=1, + nodeDescription='Dynamic single use %s slave for devstack' % machine.base_image.name, + remoteFS='/home/jenkins', + labels='%sdevstack-%s' % (DEVSTACK_GATE_PREFIX, machine.base_image.name), + launcher='hudson.plugins.sshslaves.SSHLauncher', + launcher_params = {'port': 22, 'username': 'jenkins', + 'privatekey': '/var/lib/jenkins/.ssh/id_rsa', + 'host': machine.ip}) + +def check_machine(jenkins, client, machine, error_counts): try: server = client.servers.get(machine.external_id) except: @@ -96,6 +113,8 @@ def check_machine(client, machine, error_counts): machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, 'jenkins'): + print "Adding machine %s to Jenkins" % machine.id + create_jenkins_node(jenkins, machine) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY return @@ -116,6 +135,14 @@ def check_machine(client, machine, error_counts): def main(): db = vmdatabase.VMDatabase() + config=ConfigParser.ConfigParser() + config.read(DEVSTACK_GATE_SECURE_CONFIG) + + jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), + config.get('jenkins', 'user'), + config.get('jenkins', 'apikey')) + jenkins.get_info() + provider = db.getProvider(PROVIDER_NAME) print "Working with provider %s" % provider.name @@ -156,7 +183,7 @@ def main(): print "Waiting on %s machines" % len(building_machines) for machine in building_machines: try: - check_machine(client, machine, error_counts) + check_machine(jenkins, client, machine, error_counts) except: traceback.print_exc() print "Abandoning machine %s" % machine.id diff --git a/devstack-vm-reap.py b/devstack-vm-reap.py index a2b697c5..d4d1aa22 100755 --- a/devstack-vm-reap.py +++ b/devstack-vm-reap.py @@ -23,13 +23,17 @@ import sys import time import getopt import traceback +import ConfigParser +import myjenkins import vmdatabase import utils import novaclient PROVIDER_NAME = sys.argv[1] MACHINE_LIFETIME = 24 * 60 * 60 # Amount of time after being used +DEVSTACK_GATE_SECURE_CONFIG = os.environ.get('DEVSTACK_GATE_SECURE_CONFIG', + os.path.expanduser('~/devstack-gate-secure.conf')) if '--all-servers' in sys.argv: print "Reaping all known machines" @@ -44,7 +48,7 @@ else: REAP_ALL_IMAGES = False -def delete_machine(client, machine): +def delete_machine(jenkins, client, machine): try: server = client.servers.get(machine.external_id) except novaclient.exceptions.NotFound: @@ -54,6 +58,10 @@ def delete_machine(client, machine): if server: utils.delete_server(server) + if machine.jenkins_name: + if jenkins.node_exists(machine.jenkins_name): + jenkins.delete_node(machine.jenkins_name) + machine.delete() @@ -82,6 +90,14 @@ def delete_image(client, image): def main(): db = vmdatabase.VMDatabase() + config=ConfigParser.ConfigParser() + config.read(DEVSTACK_GATE_SECURE_CONFIG) + + jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'), + config.get('jenkins', 'user'), + config.get('jenkins', 'apikey')) + jenkins.get_info() + print 'Known machines (start):' db.print_state() @@ -98,11 +114,12 @@ def main(): for machine in provider.machines: # Normally, reap machines that have sat in their current state # for 24 hours, unless that state is READY. - if REAP_ALL_SERVERS or (machine.state != vmdatabase.READY and - now - machine.state_time > MACHINE_LIFETIME): + if (REAP_ALL_SERVERS or (machine.state != vmdatabase.READY and + now - machine.state_time > MACHINE_LIFETIME) or + machine.state == vmdatabase.DELETE): print 'Deleting machine', machine.name try: - delete_machine(client, machine) + delete_machine(jenkins, client, machine) except: error = True traceback.print_exc() @@ -136,9 +153,11 @@ def main(): continue if machine.state == vmdatabase.BUILDING: continue + if machine.state == vmdatabase.HOLD: + continue print 'Deleting machine', machine.name try: - delete_machine(client, machine) + delete_machine(jenkins, client, machine) overcommitment -= 1 except: error = True diff --git a/devstack-vm-update-image.py b/devstack-vm-update-image.py index a00fbf86..1cd5a255 100755 --- a/devstack-vm-update-image.py +++ b/devstack-vm-update-image.py @@ -73,6 +73,7 @@ def tokenize(fn, tokens, distribution, comment=None): for line in open(fn): if 'dist:' in line and ('dist:%s' % distribution not in line): continue + if 'qpid' in line: continue #XXX if comment and comment in line: line = line[:line.rfind(comment)] line = line.strip() @@ -225,7 +226,8 @@ def build_image(provider, client, base_image, image, flavor, name, branches, tim # We made the snapshot, try deleting the server, but it's okay # if we fail. The reap script will find it and try again. try: - utils.delete_server(server) + pass #XXX + #utils.delete_server(server) except: print "Exception encountered deleting server:" traceback.print_exc() diff --git a/myjenkins.py b/myjenkins.py new file mode 100644 index 00000000..8c2bd8e4 --- /dev/null +++ b/myjenkins.py @@ -0,0 +1,93 @@ +import jenkins +import json +import urllib +import urllib2 +from jenkins import JenkinsException, NODE_TYPE, CREATE_NODE + +TOGGLE_OFFLINE = '/computer/%(name)s/toggleOffline?offlineMessage=%(msg)s' + +class Jenkins(jenkins.Jenkins): + def disable_node(self, name, msg=''): + ''' + Disable a node + + @param name: Jenkins node name + @type name: str + @param msg: Offline message + @type msg: str + ''' + info = self.get_node_info(name) + if info['offline']: + return + self.jenkins_open(urllib2.Request(self.server + TOGGLE_OFFLINE%locals())) + + def enable_node(self, name): + ''' + Enable a node + + @param name: Jenkins node name + @type name: str + ''' + info = self.get_node_info(name) + if not info['offline']: + return + msg = '' + self.jenkins_open(urllib2.Request(self.server + TOGGLE_OFFLINE%locals())) + + + def create_node(self, name, numExecutors=2, nodeDescription=None, + remoteFS='/var/lib/jenkins', labels=None, exclusive=False, + launcher='hudson.slaves.JNLPLauncher', launcher_params={}): + ''' + @param name: name of node to create + @type name: str + @param numExecutors: number of executors for node + @type numExecutors: int + @param nodeDescription: Description of node + @type nodeDescription: str + @param remoteFS: Remote filesystem location to use + @type remoteFS: str + @param labels: Labels to associate with node + @type labels: str + @param exclusive: Use this node for tied jobs only + @type exclusive: boolean + @param launcher: The launch method for the slave + @type launcher: str + @param launcher_params: Additional parameters for the launcher + @type launcher_params: dict + ''' + if self.node_exists(name): + raise JenkinsException('node[%s] already exists'%(name)) + + mode = 'NORMAL' + if exclusive: + mode = 'EXCLUSIVE' + + #hudson.plugins.sshslaves.SSHLauncher + #hudson.slaves.CommandLauncher + #hudson.os.windows.ManagedWindowsServiceLauncher + launcher_params['stapler-class'] = launcher + + inner_params = { + 'name' : name, + 'nodeDescription' : nodeDescription, + 'numExecutors' : numExecutors, + 'remoteFS' : remoteFS, + 'labelString' : labels, + 'mode' : mode, + 'type' : NODE_TYPE, + 'retentionStrategy' : { 'stapler-class' : 'hudson.slaves.RetentionStrategy$Always' }, + 'nodeProperties' : { 'stapler-class-bag' : 'true' }, + 'launcher' : launcher_params + } + + params = { + 'name' : name, + 'type' : NODE_TYPE, + 'json' : json.dumps(inner_params) + } + + self.jenkins_open(urllib2.Request(self.server + CREATE_NODE%urllib.urlencode(params))) + + if not self.node_exists(name): + raise JenkinsException('create[%s] failed'%(name)) diff --git a/vmdatabase.py b/vmdatabase.py index 20691f41..8849ecf1 100644 --- a/vmdatabase.py +++ b/vmdatabase.py @@ -36,6 +36,8 @@ USED = 3 ERROR = 4 # Keep this machine indefinitely HOLD = 5 +# Delete this machine immediately (probably a used machine) +DELETE = 6 # Possible Jenkins results RESULT_SUCCESS = 1 @@ -84,12 +86,13 @@ snapshot_image_table = Table('snapshot_image', metadata, machine_table = Table('machine', metadata, Column('id', Integer, primary_key=True), Column('base_image_id', Integer, ForeignKey('base_image.id'), index=True, nullable=False), - Column('external_id', String(255)), # Provider assigned id for this machine - Column('name', String(255)), # Machine name - Column('ip', String(255)), # Primary IP address - Column('user', String(255)), # Username if ssh keys have been installed, or NULL - Column('state', Integer), # One of the above values - Column('state_time', Integer), # Time of last state change + Column('external_id', String(255)), # Provider assigned id for this machine + Column('name', String(255)), # Machine name + Column('jenkins_name', String(255)), # Jenkins node name + Column('ip', String(255)), # Primary IP address + Column('user', String(255)), # Username if ssh keys have been installed, or NULL + Column('state', Integer), # One of the above values + Column('state_time', Integer), # Time of last state change ) result_table = Table('result', metadata, Column('id', Integer, primary_key=True), @@ -359,6 +362,9 @@ class VMDatabase(object): def getMachine(self, id): return self.session.query(Machine).filter_by(id=id)[0] + def getMachineByJenkinsName(self, name): + return self.session.query(Machine).filter_by(jenkins_name=name)[0] + def getMachineForUse(self, image_name): """Atomically find a machine that is ready for use, and update its state."""