Several improvements to integration testing.

Change libcloud imports to remove deprecated module paths.

Actually move the pip cache into the place devstack expects on
the devstack host.

As soon as the new version of openstack-ci is checked out, start
using it for the scripts in the currently running test, so that
we are testing as much of the test infrastructure itself as
possible.

Allow the update and launch scripts to have the server/image name
specified in an environment variable for easier
testing/upgrading.

Change-Id: Iee0a946afa80929e99c08e5a6b0504c1d8dd0093

slave_scripts/devstack-vm-delete.py

@@ -18,11 +18,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from libcloud.base import NodeImage, NodeSize, NodeLocation
-from libcloud.types import Provider
-from libcloud.providers import get_driver
-from libcloud.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
-import libcloud
+from libcloud.compute.base import NodeImage, NodeSize, NodeLocation
+from libcloud.compute.types import Provider
+from libcloud.compute.providers import get_driver
 import os, sys
 import getopt
 import time

slave_scripts/devstack-vm-gate-host.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
 
 # Script that is run on the devstack vm; configures and 
 # invokes devstack.
@@ -59,6 +59,10 @@ if ls ~/cache/files/*; then
     mv ~/cache/files/* /opt/stack/devstack/files
 fi
 
+# Move the PIP cache into position:
+sudo mkdir -p /var/cache/pip
+sudo mv ~/cache/pip/* /var/cache/pip
+
 # Start with a fresh syslog
 sudo stop rsyslog
 sudo mv /var/log/syslog /var/log/syslog-pre-devstack

slave_scripts/devstack-vm-gate.sh

@@ -24,7 +24,6 @@ PROJECTS="openstack/nova openstack/glance openstack/keystone openstack/python-no
 # Set this to 1 to always keep the host around
 ALWAYS_KEEP=${ALWAYS_KEEP:-0}
 
-CI_SCRIPT_DIR=$(cd $(dirname "$0") && pwd)
 cd $WORKSPACE
 mkdir -p logs
 rm -f logs/*
@@ -63,6 +62,10 @@ do
     cd $WORKSPACE
 done
 
+# Set CI_SCRIPT_DIR to point to opestack-ci in the workspace so that
+# we are testing the proposed change from this point forward.
+CI_SCRIPT_DIR=$WORKSPACE/openstack-ci/slave-scripts
+
 eval `$CI_SCRIPT_DIR/devstack-vm-fetch.py` || exit $?
 
 scp -C $CI_SCRIPT_DIR/devstack-vm-gate-host.sh $NODE_IP_ADDR:

slave_scripts/devstack-vm-launch.py

@@ -19,11 +19,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from libcloud.base import NodeImage, NodeSize, NodeLocation
-from libcloud.compute.types import NodeState
-from libcloud.types import Provider
-from libcloud.providers import get_driver
-from libcloud.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
+from libcloud.compute.base import NodeImage, NodeSize, NodeLocation
+from libcloud.compute.types import Provider, NodeState
+from libcloud.compute.providers import get_driver
+from libcloud.compute.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
+
 import libcloud
 import os, sys
 import getopt
@@ -34,7 +34,8 @@ import vmdatabase
 CLOUD_SERVERS_DRIVER = os.environ.get('CLOUD_SERVERS_DRIVER','rackspace')
 CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME']
 CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY']
-IMAGE_NAME = 'devstack-oneiric'
+IMAGE_NAME = os.environ.get('IMAGE_NAME', 'devstack-oneiric')
+
 MIN_RAM = 1024
 MIN_READY_MACHINES = 5
 

slave_scripts/devstack-vm-reap.py

@@ -21,11 +21,9 @@
 import os, sys, time
 import getopt
 
-from libcloud.base import NodeImage, NodeSize, NodeLocation
-from libcloud.types import Provider
-from libcloud.providers import get_driver
-from libcloud.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment
-import libcloud
+from libcloud.compute.base import NodeImage, NodeSize, NodeLocation
+from libcloud.compute.types import Provider
+from libcloud.compute.providers import get_driver
 
 import vmdatabase
 

slave_scripts/devstack-vm-update-image.py

@@ -38,9 +38,9 @@ CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME']
 CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY']
 WORKSPACE = os.environ['WORKSPACE']
 DEVSTACK = os.path.join(WORKSPACE, 'devstack')
-
-SERVER_NAME = 'devstack-oneiric.template.openstack.org'
-IMAGE_NAME = 'devstack-oneiric'
+SERVER_NAME = os.environ.get('SERVER_NAME',
+                             'devstack-oneiric.template.openstack.org')
+IMAGE_NAME = os.environ.get('IMAGE_NAME', 'devstack-oneiric')
 DISTRIBUTION = 'oneiric'
 
 def run_local(cmd, status=False, cwd='.', env={}):