Cache downloaded pips/debs on image.

Cache the specified pip and deb packages on the image for
all branches.  This should mean that the image has all of the
dependencies cached locally ready to install for whatever
branch devstack will run from.

Change-Id: Ib15ba886cbda14e757ac88e5b3f858fff98cd79d

slave_scripts/devstack-vm-gate-host.sh

@@ -54,7 +54,9 @@ EOF
 
 # The vm template update job should cache some images in ~/files.
 # Move them to where devstack expects:
-mv ~/files/* /opt/stack/devstack/files
+if ls ~/cache/files/*; then
+    mv ~/cache/files/* /opt/stack/devstack/files
+fi
 
 ./stack.sh
 ./exercise.sh

slave_scripts/devstack-vm-update-image.py

@@ -31,17 +31,79 @@ import os
 import commands
 import time
 import paramiko
+import subprocess
 
 CLOUD_SERVERS_DRIVER = os.environ.get('CLOUD_SERVERS_DRIVER','rackspace')
 CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME']
 CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY']
+WORKSPACE = os.environ['WORKSPACE']
+DEVSTACK = os.path.join(WORKSPACE, 'devstack')
 
 SERVER_NAME = 'devstack-oneiric.template.openstack.org'
 IMAGE_NAME = 'devstack-oneiric'
+DISTRIBUTION = 'oneiric'
 
-debs = ' '.join(open(sys.argv[1]).read().split('\n'))
-pips = ' '.join(open(sys.argv[2]).read().split('\n'))
-filedir = sys.argv[3]
+def run_local(cmd, status=False, cwd='.', env={}):
+    print "Running:", cmd
+    newenv = os.environ
+    newenv.update(env)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
+                         stderr=subprocess.STDOUT, env=newenv)
+    (out, nothing) = p.communicate()
+    if status:
+        return (p.returncode, out.strip())
+    return out.strip()
+
+def git_branches():
+    branches = []
+    for branch in run_local(['git', 'branch'], cwd=DEVSTACK).split("\n"):
+        if branch.startswith('*'):
+            branch = branch.split()[1]
+        branches.append(branch.strip())
+    return branches
+
+def tokenize(fn, tokens, comment=None):
+    for line in open(fn):
+        if 'dist:' in line and ('dist:%s'%DISTRIBUTION not in line):
+            continue
+        if comment and comment in line:
+            line = line[:line.rfind(comment)]
+        line = line.strip()
+        if line and line not in tokens:
+            tokens.append(line)
+
+BRANCHES = []
+for branch in git_branches():
+    branch_data = {'name': branch}
+    print 'Branch: ', branch
+    run_local(['git', 'checkout', branch], cwd=DEVSTACK)
+    run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
+
+    pips = []
+    pipdir = os.path.join(DEVSTACK, 'files', 'pips')
+    for fn in os.listdir(pipdir):
+        fn = os.path.join(pipdir, fn)
+        tokenize(fn, pips)
+    branch_data['pips'] = pips
+
+    debs = []
+    debdir = os.path.join(DEVSTACK, 'files', 'apts')
+    for fn in os.listdir(debdir):
+        fn = os.path.join(debdir, fn)
+        tokenize(fn, debs, comment='#')
+    branch_data['debs'] = debs
+
+    images = []
+    for line in open(os.path.join(DEVSTACK, 'stackrc')):
+        if line.startswith('IMAGE_URLS'):
+            if '#' in line: 
+                line = line[:line.rfind('#')]
+            value = line.split('=', 1)[1].strip()
+            if value[0]==value[-1]=='"':
+                value=value[1:-1]
+            images += [x.strip() for x in value.split(',')]
+    branch_data['images'] = images
+    BRANCHES.append(branch_data)
 
 if CLOUD_SERVERS_DRIVER == 'rackspace':
     Driver = get_driver(Provider.RACKSPACE)
@@ -62,14 +124,19 @@ client.load_system_host_keys()
 client.set_missing_host_key_policy(paramiko.WarningPolicy())
 client.connect(ip)
 
-def run(action, x):
+def ssh(action, x):
     stdin, stdout, stderr = client.exec_command(x)
     print x
+    output = ''
+    for x in stdout:
+        output += x
+        sys.stdout.write(x)
+        sys.stdout.flush()
     ret = stdout.channel.recv_exit_status()
-    print stdout.read()
     print stderr.read()
     if ret:
         raise Exception("Unable to %s" % action)
+    return output
 
 def scp(source, dest):
     print 'copy', source, dest
@@ -77,17 +144,37 @@ def scp(source, dest):
     ftp.put(source, dest)
     ftp.close()
 
-run('make file cache directory', 'mkdir -p files')
-for fn in os.listdir(filedir):
-    source = os.path.join(filedir, fn)
-    scp(source, 'files/%s'%fn)
-run('update package list', 'sudo apt-get update')
-run('install packages', 'sudo DEBIAN_FRONTEND=noninteractive apt-get --option "Dpkg::Options::=--force-confold" --assume-yes install %s' % debs)
-run('install pips', 'sudo pip install %s' % pips)
-run('upgrade server', 'sudo apt-get -y dist-upgrade')
-run('run puppet', 'sudo bash -c "cd /root/openstack-ci-puppet && /usr/bin/git pull -q && /var/lib/gems/1.8/bin/puppet apply -l /tmp/manifest.log --modulepath=/root/openstack-ci-puppet/modules manifests/site.pp"')
-run('stop mysql', 'sudo /etc/init.d/mysql stop')
-run('stop rabbitmq', 'sudo /etc/init.d/rabbitmq-server stop')
+ssh('make file cache directory', 'mkdir -p ~/cache/files')
+ssh('make pip cache directory', 'mkdir -p ~/cache/pip')
+ssh('update package list', 'sudo apt-get update')
+ssh('upgrade server', 'sudo apt-get -y dist-upgrade')
+ssh('run puppet', 'sudo bash -c "cd /root/openstack-ci-puppet && /usr/bin/git pull -q && /var/lib/gems/1.8/bin/puppet apply -l /tmp/manifest.log --modulepath=/root/openstack-ci-puppet/modules manifests/site.pp"')
+
+for branch_data in BRANCHES:
+    ssh('cache debs for branch %s'%branch_data['name'], 
+        'sudo apt-get -y -d install %s' % ' '.join(branch_data['debs']))
+    venv = ssh('get temp dir for venv', 'mktemp -d').strip()
+    ssh('create venv', 'virtualenv --no-site-packages %s' % venv)
+    ssh('cache pips for branch %s'%branch_data['name'], 
+        'source %s/bin/activate && PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' % (venv, ' '.join(branch_data['pips'])))
+    ssh('remove venv', 'rm -fr %s'%venv)
+    for url in branch_data['images']:
+        fname = url.split('/')[-1]
+        try:
+            ssh('check for %s'%fname, 'ls ~/cache/files/%s'%fname)
+        except:
+            ssh('download image %s'%fname,
+                'wget -c %s -O ~/cache/files/%s' % (url, fname))
+
+# TODO: remove after mysql/rabbitmq are removed from image
+try:
+    ssh('stop mysql', 'sudo /etc/init.d/mysql stop')
+except:
+    pass
+try:
+    ssh('stop rabbitmq', 'sudo /etc/init.d/rabbitmq-server stop')
+except:
+    pass
 
 for image in images:
     print 'Deleting image', image

slave_scripts/devstack-vm-update-image.sh

@@ -26,23 +26,6 @@ if [[ ! -e devstack ]]; then
 fi
 cd devstack
 git remote update
-git pull --ff-only origin
-git checkout $GERRIT_BRANCH
-
 cd $WORKSPACE
-cat devstack/files/apts/* | grep -v NOPRIME | cut -d\# -f1 > devstack-debs
-cat devstack/files/pips/* > devstack-pips
 
-source $WORKSPACE/devstack/stackrc
-mkdir -p files
-# Excerpt from devstack that downloads the images
-for image_url in ${IMAGE_URLS//,/ }; do
-    # Downloads the image (uec ami+aki style), then extracts it.
-    IMAGE_FNAME=`echo "$image_url" | python -c "import sys; print sys.stdin.read().split('/')[-1]"`
-    IMAGE_NAME=`echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]"`
-    if [ ! -f files/$IMAGE_FNAME ]; then
-        wget -c $image_url -O files/$IMAGE_FNAME
-    fi
-done
-
-$CI_SCRIPT_DIR/devstack-vm-update-image.py $WORKSPACE/devstack-debs $WORKSPACE/devstack-pips $WORKSPACE/files
+$CI_SCRIPT_DIR/devstack-vm-update-image.py