diff --git a/slave_scripts/devstack-vm-gate-host.sh b/slave_scripts/devstack-vm-gate-host.sh index 42c1592b..3a172b51 100755 --- a/slave_scripts/devstack-vm-gate-host.sh +++ b/slave_scripts/devstack-vm-gate-host.sh @@ -54,7 +54,9 @@ EOF # The vm template update job should cache some images in ~/files. # Move them to where devstack expects: -mv ~/files/* /opt/stack/devstack/files +if ls ~/cache/files/*; then + mv ~/cache/files/* /opt/stack/devstack/files +fi ./stack.sh ./exercise.sh diff --git a/slave_scripts/devstack-vm-update-image.py b/slave_scripts/devstack-vm-update-image.py index e1a34330..ee3b237d 100755 --- a/slave_scripts/devstack-vm-update-image.py +++ b/slave_scripts/devstack-vm-update-image.py @@ -31,17 +31,79 @@ import os import commands import time import paramiko +import subprocess CLOUD_SERVERS_DRIVER = os.environ.get('CLOUD_SERVERS_DRIVER','rackspace') CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME'] CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY'] +WORKSPACE = os.environ['WORKSPACE'] +DEVSTACK = os.path.join(WORKSPACE, 'devstack') SERVER_NAME = 'devstack-oneiric.template.openstack.org' IMAGE_NAME = 'devstack-oneiric' +DISTRIBUTION = 'oneiric' -debs = ' '.join(open(sys.argv[1]).read().split('\n')) -pips = ' '.join(open(sys.argv[2]).read().split('\n')) -filedir = sys.argv[3] +def run_local(cmd, status=False, cwd='.', env={}): + print "Running:", cmd + newenv = os.environ + newenv.update(env) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd, + stderr=subprocess.STDOUT, env=newenv) + (out, nothing) = p.communicate() + if status: + return (p.returncode, out.strip()) + return out.strip() + +def git_branches(): + branches = [] + for branch in run_local(['git', 'branch'], cwd=DEVSTACK).split("\n"): + if branch.startswith('*'): + branch = branch.split()[1] + branches.append(branch.strip()) + return branches + +def tokenize(fn, tokens, comment=None): + for line in open(fn): + if 'dist:' in line and ('dist:%s'%DISTRIBUTION not in line): + continue + if comment and comment in line: + line = line[:line.rfind(comment)] + line = line.strip() + if line and line not in tokens: + tokens.append(line) + +BRANCHES = [] +for branch in git_branches(): + branch_data = {'name': branch} + print 'Branch: ', branch + run_local(['git', 'checkout', branch], cwd=DEVSTACK) + run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) + + pips = [] + pipdir = os.path.join(DEVSTACK, 'files', 'pips') + for fn in os.listdir(pipdir): + fn = os.path.join(pipdir, fn) + tokenize(fn, pips) + branch_data['pips'] = pips + + debs = [] + debdir = os.path.join(DEVSTACK, 'files', 'apts') + for fn in os.listdir(debdir): + fn = os.path.join(debdir, fn) + tokenize(fn, debs, comment='#') + branch_data['debs'] = debs + + images = [] + for line in open(os.path.join(DEVSTACK, 'stackrc')): + if line.startswith('IMAGE_URLS'): + if '#' in line: + line = line[:line.rfind('#')] + value = line.split('=', 1)[1].strip() + if value[0]==value[-1]=='"': + value=value[1:-1] + images += [x.strip() for x in value.split(',')] + branch_data['images'] = images + BRANCHES.append(branch_data) if CLOUD_SERVERS_DRIVER == 'rackspace': Driver = get_driver(Provider.RACKSPACE) @@ -62,14 +124,19 @@ client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy()) client.connect(ip) -def run(action, x): +def ssh(action, x): stdin, stdout, stderr = client.exec_command(x) print x + output = '' + for x in stdout: + output += x + sys.stdout.write(x) + sys.stdout.flush() ret = stdout.channel.recv_exit_status() - print stdout.read() print stderr.read() if ret: raise Exception("Unable to %s" % action) + return output def scp(source, dest): print 'copy', source, dest @@ -77,17 +144,37 @@ def scp(source, dest): ftp.put(source, dest) ftp.close() -run('make file cache directory', 'mkdir -p files') -for fn in os.listdir(filedir): - source = os.path.join(filedir, fn) - scp(source, 'files/%s'%fn) -run('update package list', 'sudo apt-get update') -run('install packages', 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes install %s' % debs) -run('install pips', 'sudo pip install %s' % pips) -run('upgrade server', 'sudo apt-get -y dist-upgrade') -run('run puppet', 'sudo bash -c "cd /root/openstack-ci-puppet && /usr/bin/git pull -q && /var/lib/gems/1.8/bin/puppet apply -l /tmp/manifest.log --modulepath=/root/openstack-ci-puppet/modules manifests/site.pp"') -run('stop mysql', 'sudo /etc/init.d/mysql stop') -run('stop rabbitmq', 'sudo /etc/init.d/rabbitmq-server stop') +ssh('make file cache directory', 'mkdir -p ~/cache/files') +ssh('make pip cache directory', 'mkdir -p ~/cache/pip') +ssh('update package list', 'sudo apt-get update') +ssh('upgrade server', 'sudo apt-get -y dist-upgrade') +ssh('run puppet', 'sudo bash -c "cd /root/openstack-ci-puppet && /usr/bin/git pull -q && /var/lib/gems/1.8/bin/puppet apply -l /tmp/manifest.log --modulepath=/root/openstack-ci-puppet/modules manifests/site.pp"') + +for branch_data in BRANCHES: + ssh('cache debs for branch %s'%branch_data['name'], + 'sudo apt-get -y -d install %s' % ' '.join(branch_data['debs'])) + venv = ssh('get temp dir for venv', 'mktemp -d').strip() + ssh('create venv', 'virtualenv --no-site-packages %s' % venv) + ssh('cache pips for branch %s'%branch_data['name'], + 'source %s/bin/activate && PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' % (venv, ' '.join(branch_data['pips']))) + ssh('remove venv', 'rm -fr %s'%venv) + for url in branch_data['images']: + fname = url.split('/')[-1] + try: + ssh('check for %s'%fname, 'ls ~/cache/files/%s'%fname) + except: + ssh('download image %s'%fname, + 'wget -c %s -O ~/cache/files/%s' % (url, fname)) + +# TODO: remove after mysql/rabbitmq are removed from image +try: + ssh('stop mysql', 'sudo /etc/init.d/mysql stop') +except: + pass +try: + ssh('stop rabbitmq', 'sudo /etc/init.d/rabbitmq-server stop') +except: + pass for image in images: print 'Deleting image', image diff --git a/slave_scripts/devstack-vm-update-image.sh b/slave_scripts/devstack-vm-update-image.sh index 412f05e9..fc85e293 100755 --- a/slave_scripts/devstack-vm-update-image.sh +++ b/slave_scripts/devstack-vm-update-image.sh @@ -26,23 +26,6 @@ if [[ ! -e devstack ]]; then fi cd devstack git remote update -git pull --ff-only origin -git checkout $GERRIT_BRANCH - cd $WORKSPACE -cat devstack/files/apts/* | grep -v NOPRIME | cut -d\# -f1 > devstack-debs -cat devstack/files/pips/* > devstack-pips -source $WORKSPACE/devstack/stackrc -mkdir -p files -# Excerpt from devstack that downloads the images -for image_url in ${IMAGE_URLS//,/ }; do - # Downloads the image (uec ami+aki style), then extracts it. - IMAGE_FNAME=`echo "$image_url" | python -c "import sys; print sys.stdin.read().split('/')[-1]"` - IMAGE_NAME=`echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]"` - if [ ! -f files/$IMAGE_FNAME ]; then - wget -c $image_url -O files/$IMAGE_FNAME - fi -done - -$CI_SCRIPT_DIR/devstack-vm-update-image.py $WORKSPACE/devstack-debs $WORKSPACE/devstack-pips $WORKSPACE/files +$CI_SCRIPT_DIR/devstack-vm-update-image.py