diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..f3d74a9a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +*~ diff --git a/devstack-vm-delete.py b/devstack-vm-delete.py index d1df4bd8..1707c8cc 100755 --- a/devstack-vm-delete.py +++ b/devstack-vm-delete.py @@ -2,7 +2,7 @@ # Delete a devstack VM. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,27 +18,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -from libcloud.compute.base import NodeImage, NodeSize, NodeLocation -from libcloud.compute.types import Provider -from libcloud.compute.providers import get_driver -import os, sys +import os +import sys import getopt import time import vmdatabase +import utils -CLOUD_SERVERS_DRIVER = os.environ.get('CLOUD_SERVERS_DRIVER','rackspace') -CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME'] -CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY'] +NODE_ID = sys.argv[1] -node_uuid = sys.argv[1] -db = vmdatabase.VMDatabase() -machine = db.getMachine(node_uuid) -if CLOUD_SERVERS_DRIVER == 'rackspace': - Driver = get_driver(Provider.RACKSPACE) - conn = Driver(CLOUD_SERVERS_USERNAME, CLOUD_SERVERS_API_KEY) - node = [n for n in conn.list_nodes() if n.id==str(machine['id'])][0] - node.destroy() +def main(): + db = vmdatabase.VMDatabase() + machine = db.getMachine(NODE_ID) + provider = machine.base_image.provider -db.delMachine(node_uuid) + client = utils.get_client(provider) + + server = client.servers.get(machine.external_id) + utils.delete_server(server) + machine.delete() + + +if __name__ == '__main__': + main() diff --git a/devstack-vm-fetch.py b/devstack-vm-fetch.py index 68110906..858bbb55 100755 --- a/devstack-vm-fetch.py +++ b/devstack-vm-fetch.py @@ -2,7 +2,7 @@ # Fetch a ready VM for use by devstack. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,13 +18,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + import vmdatabase +IMAGE_NAME = sys.argv[1] + db = vmdatabase.VMDatabase() -node = db.getMachineForUse() +node = db.getMachineForUse(IMAGE_NAME) if not node: raise Exception("No ready nodes") -print "NODE_IP_ADDR=%s" % node['ip'] -print "NODE_UUID=%s" % node['uuid'] +print "NODE_IP_ADDR=%s" % node.ip +print "NODE_PROVIDER=%s" % node.base_image.provider.name +print "NODE_ID=%s" % node.id diff --git a/devstack-vm-gate-host.sh b/devstack-vm-gate-host.sh index 305c14af..7d587e99 100755 --- a/devstack-vm-gate-host.sh +++ b/devstack-vm-gate-host.sh @@ -3,7 +3,7 @@ # Script that is run on the devstack vm; configures and # invokes devstack. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,6 +34,15 @@ if [ ! -w $DEST ]; then sudo chown `whoami` $DEST fi +# Hpcloud provides no swap, but does have a partition mounted at /mnt +# we can use: +if [ `cat /proc/meminfo | grep SwapTotal | awk '{ print $2; }'` -eq 0 ] && + [ -b /dev/vdb ]; then + sudo umount /dev/vdb + sudo mkswap /dev/vdb + sudo swapon /dev/vdb +fi + # The workspace has been copied over here by devstack-vm-gate.sh mv * /opt/stack cd /opt/stack/devstack @@ -52,6 +61,8 @@ SKIP_EXERCISES=boot_from_volume,client-env,swift SERVICE_HOST=127.0.0.1 SYSLOG=True SCREEN_LOGDIR=/opt/stack/screen-logs +FIXED_RANGE=10.1.0.0/24 +FIXED_NETWORK_SIZE=256 EOF # The vm template update job should cache some images in ~/files. diff --git a/devstack-vm-gate.sh b/devstack-vm-gate.sh index 1dd42858..78437b4b 100755 --- a/devstack-vm-gate.sh +++ b/devstack-vm-gate.sh @@ -3,7 +3,7 @@ # Gate commits to several projects on a VM running those projects # configured by devstack. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +19,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -PROJECTS="openstack-ci/devstack-gate openstack-dev/devstack openstack/nova openstack/glance openstack/keystone openstack/python-novaclient openstack/python-keystoneclient openstack/python-quantumclient openstack/horizon" +PROJECTS="openstack-dev/devstack openstack/nova openstack/glance openstack/keystone openstack/python-novaclient openstack/python-keystoneclient openstack/python-quantumclient openstack/horizon" + +# Set this variable to skip updating the devstack-gate project itself. +# Useful in development so you can edit scripts in place and run them +# directly. Do not set in production. +# Normally not set, and we do include devstack-gate with the rest of +# the projects. +if [ -z $SKIP_DEVSTACK_GATE_PROJECT ] +then + PROJECTS="openstack-ci/devstack-gate $PROJECTS" +fi # Set this to 1 to always keep the host around ALWAYS_KEEP=${ALWAYS_KEEP:-0} @@ -75,14 +85,14 @@ if [[ $GERRIT_PROJECT == "openstack-ci/devstack-gate" ]] && [[ $RE_EXEC != "true exec $GATE_SCRIPT_DIR/devstack-vm-gate.sh fi -FETCH_OUTPUT=`$GATE_SCRIPT_DIR/devstack-vm-fetch.py` || exit $? +FETCH_OUTPUT=`$GATE_SCRIPT_DIR/devstack-vm-fetch.py oneiric` || exit $? eval $FETCH_OUTPUT scp -C $GATE_SCRIPT_DIR/devstack-vm-gate-host.sh $NODE_IP_ADDR: RETVAL=$? if [ $RETVAL != 0 ]; then echo "Deleting host" - $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_UUID + $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID exit $RETVAL fi @@ -90,7 +100,7 @@ rsync -az --delete $WORKSPACE/ $NODE_IP_ADDR:workspace/ RETVAL=$? if [ $RETVAL != 0 ]; then echo "Deleting host" - $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_UUID + $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID exit $RETVAL fi @@ -106,10 +116,10 @@ rm $WORKSPACE/logs/*.*.txt # Now check whether the run was a success if [ $RETVAL = 0 ] && [ $ALWAYS_KEEP = 0 ]; then echo "Deleting host" - $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_UUID + $GATE_SCRIPT_DIR/devstack-vm-delete.py $NODE_ID exit $RETVAL else #echo "Giving host to developer" - #$GATE_SCRIPT_DIR/devstack-vm-give.py $NODE_UUID + #$GATE_SCRIPT_DIR/devstack-vm-give.py $NODE_ID exit $RETVAL fi diff --git a/devstack-vm-give.py b/devstack-vm-give.py index 8e68cc1d..c7987fd8 100755 --- a/devstack-vm-give.py +++ b/devstack-vm-give.py @@ -3,7 +3,7 @@ # Turn over a devstack configured machine to the developer who # proposed the change that is being tested. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +19,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, sys, time +import os +import sys +import time import getopt import commands import json @@ -28,41 +30,50 @@ import tempfile import vmdatabase -node_uuid = sys.argv[1] -db = vmdatabase.VMDatabase() -machine = db.getMachine(node_uuid) +NODE_ID = sys.argv[1] -stat, out = commands.getstatusoutput("ssh -p 29418 review.openstack.org gerrit query --format=JSON change:%s" % os.environ['GERRIT_CHANGE_NUMBER']) -data = json.loads(out.split('\n')[0]) -username = data['owner']['username'] +def main(): + db = vmdatabase.VMDatabase() + machine = db.getMachine(NODE_ID) -f = urllib2.urlopen('https://launchpad.net/~%s/+sshkeys'%username) -keys = f.read() + stat, out = commands.getstatusoutput( + "ssh -p 29418 review.openstack.org gerrit" + + "query --format=JSON change:%s" % + os.environ['GERRIT_CHANGE_NUMBER']) -tmp = tempfile.NamedTemporaryFile(delete=False) -try: - tmp.write("""#!/bin/bash + data = json.loads(out.split('\n')[0]) + username = data['owner']['username'] + + f = urllib2.urlopen('https://launchpad.net/~%s/+sshkeys' % username) + keys = f.read() + + tmp = tempfile.NamedTemporaryFile(delete=False) + try: + tmp.write("""#!/bin/bash chmod u+w ~/.ssh/authorized_keys cat <>~/.ssh/authorized_keys """) - tmp.write(keys) - tmp.write("\nEOF\n") - tmp.close() - stat, out = commands.getstatusoutput("scp %s %s:/var/tmp/keys.sh" % - (tmp.name, machine['ip'])) - if stat: - print out - raise Exception("Unable to copy keys") + tmp.write(keys) + tmp.write("\nEOF\n") + tmp.close() + stat, out = commands.getstatusoutput("scp %s %s:/var/tmp/keys.sh" % + (tmp.name, machine.ip)) + if stat: + print out + raise Exception("Unable to copy keys") - stat, out = commands.getstatusoutput("ssh %s /bin/sh /var/tmp/keys.sh" % - machine['ip']) - - if stat: - print out - raise Exception("Unable to add keys") -finally: - os.unlink(tmp.name) + stat, out = commands.getstatusoutput( + "ssh %s /bin/sh /var/tmp/keys.sh" % machine.ip) -db.setMachineUser(machine['id'], username) -print "Added %s to authorized_keys on %s" % (username, machine['ip']) + if stat: + print out + raise Exception("Unable to add keys") + finally: + os.unlink(tmp.name) + + machine.user = username + print "Added %s to authorized_keys on %s" % (username, machine.ip) + +if __name__ == '__main__': + main() diff --git a/devstack-vm-launch.py b/devstack-vm-launch.py index ec3654b6..233c294f 100755 --- a/devstack-vm-launch.py +++ b/devstack-vm-launch.py @@ -3,7 +3,7 @@ # Make sure there are always a certain number of VMs launched and # ready for use by devstack. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,154 +19,153 @@ # See the License for the specific language governing permissions and # limitations under the License. -from libcloud.compute.base import NodeImage, NodeSize, NodeLocation -from libcloud.compute.types import Provider, NodeState -from libcloud.compute.providers import get_driver -from libcloud.compute.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment - -import libcloud -import os, sys +import os +import sys import getopt import time import paramiko import traceback import vmdatabase +import utils -CLOUD_SERVERS_DRIVER = os.environ.get('CLOUD_SERVERS_DRIVER','rackspace') -CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME'] -CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY'] -IMAGE_NAME = os.environ.get('IMAGE_NAME', 'devstack-oneiric') +PROVIDER_NAME = sys.argv[1] +DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '') -MIN_RAM = 1024 -MIN_READY_MACHINES = 10 # keep this number of machine in the pool ABANDON_TIMEOUT = 900 # assume a machine will never boot if it hasn't # after this amount of time -db = vmdatabase.VMDatabase() -ready_machines = [x for x in db.getMachines() - if x['state'] == vmdatabase.READY] -building_machines = [x for x in db.getMachines() - if x['state'] == vmdatabase.BUILDING] +def calculate_deficit(provider, base_image): + # Count machines that are ready and machines that are building, + # so that if the provider is very slow, we aren't queueing up tons + # of machines to be built. + num_to_launch = base_image.min_ready - (len(base_image.ready_machines) + + len(base_image.building_machines)) -# Count machines that are ready and machines that are building, -# so that if the provider is very slow, we aren't queueing up tons -# of machines to be built. -num_to_launch = MIN_READY_MACHINES - (len(ready_machines) + - len(building_machines)) + # Don't launch more than our provider max + num_to_launch = min(provider.max_servers - len(provider.machines), + num_to_launch) -print "%s ready, %s building, need to launch %s" % (len(ready_machines), - len(building_machines), - num_to_launch) + # Don't launch less than 0 + num_to_launch = max(0, num_to_launch) -if num_to_launch <= 0 and len(building_machines) == 0: - sys.exit(0) + print "Ready nodes: ", len(base_image.ready_machines) + print "Building nodes:", len(base_image.building_machines) + print "Provider total:", len(provider.machines) + print "Provider max: ", provider.max_servers + print "Need to launch:", num_to_launch -if CLOUD_SERVERS_DRIVER == 'rackspace': - Driver = get_driver(Provider.RACKSPACE) - conn = Driver(CLOUD_SERVERS_USERNAME, CLOUD_SERVERS_API_KEY) - images = conn.list_images() + return num_to_launch - sizes = [sz for sz in conn.list_sizes() if sz.ram >= MIN_RAM] - sizes.sort(lambda a,b: cmp(a.ram, b.ram)) - size = sizes[0] - images = [img for img in conn.list_images() - if img.name.startswith(IMAGE_NAME)] - images.sort() - if not len(images): - raise Exception("No images found") - image = images[-1] -else: - raise Exception ("Driver not supported") -def check_ssh(ip): - client = paramiko.SSHClient() - client.load_system_host_keys() - client.set_missing_host_key_policy(paramiko.WarningPolicy()) - client.connect(ip, timeout=10) +def launch_node(client, snap_image, image, flavor, last_name): + while True: + name = '%sdevstack-%s.slave.openstack.org' % ( + DEVSTACK_GATE_PREFIX, int(time.time())) + if name != last_name: + break + time.sleep(1) + create_kwargs = dict(image=image, flavor=flavor, name=name) + server = client.servers.create(**create_kwargs) + machine = snap_image.base_image.newMachine(name=name, + external_id=server.id) + print "Started building machine %s:" % machine.id + print " name: %s" % (name) + print + return server, machine - stdin, stdout, stderr = client.exec_command("echo SSH check succeeded") - print stdout.read() - print stderr.read() - ret = stdout.channel.recv_exit_status() - if ret: - raise Exception("Echo command failed") - return True -if CLOUD_SERVERS_DRIVER == 'rackspace': +def check_machine(client, machine, error_counts): + try: + server = client.servers.get(machine.external_id) + except: + print "Unable to get server detail, will retry" + traceback.print_exc() + return + + if server.status == 'ACTIVE': + if 'os-floating-ips' in utils.get_extensions(client): + utils.add_public_ip(server) + ip = utils.get_public_ip(server) + if not ip: + raise Exception("Unable to find public ip of server") + machine.ip = ip + print "Machine %s is running, testing ssh" % machine.id + if utils.ssh_connect(ip, 'jenkins'): + print "Machine %s is ready" % machine.id + machine.state = vmdatabase.READY + return + elif not server.status.startswith('BUILD'): + count = error_counts.get(machine.id, 0) + count += 1 + error_counts[machine.id] = count + print "Machine %s is in error %s (%s/5)" % (machine.id, + server.status, + count) + if count >= 5: + raise Exception("Too many errors querying machine %s" % machine.id) + else: + if time.time() - machine.state_time >= ABANDON_TIMEOUT: + raise Exception("Waited too long for machine %s" % machine.id) + + +def main(): + db = vmdatabase.VMDatabase() + + provider = db.getProvider(PROVIDER_NAME) + print "Working with provider %s" % provider.name + + client = utils.get_client(provider) + last_name = '' error_counts = {} - for i in range(num_to_launch): - while True: - node_name = 'devstack-%s.slave.openstack.org' % int(time.time()) - if node_name != last_name: break - time.sleep(1) - node = conn.create_node(name=node_name, image=image, size=size) - db.addMachine(CLOUD_SERVERS_DRIVER, node.id, IMAGE_NAME, - node_name, node.public_ip[0], node.uuid) - print "Started building node %s:" % node.id - print " name: %s [%s]" % (node_name, node.public_ip[0]) - print " uuid: %s" % (node.uuid) - print - - # Wait for nodes - # TODO: The vmdatabase is (probably) ready, but this needs reworking to - # actually support multiple providers - start = time.time() - to_ignore = [] error = False - while True: - building_machines = [x for x in db.getMachines() - if x['state'] == vmdatabase.BUILDING] - if not building_machines: - print "Finished" - break - try: - provider_nodes = conn.list_nodes() - except Exception, e: - traceback.print_exc() - print "Unable to list nodes" + + for base_image in provider.base_images: + snap_image = base_image.current_snapshot + if not snap_image: continue + print "Working on image %s" % snap_image.name + + flavor = utils.get_flavor(client, base_image.min_ram) + print "Found flavor", flavor + + remote_snap_image = client.images.get(snap_image.external_id) + print "Found image", remote_snap_image + + num_to_launch = calculate_deficit(provider, base_image) + for i in range(num_to_launch): + try: + server, machine = launch_node(client, snap_image, + remote_snap_image, flavor, last_name) + last_name = machine.name + except: + traceback.print_exc() + error = True + + while True: + building_machines = provider.building_machines + if not building_machines: + print "No more machines are building, finished." + break + print "Waiting on %s machines" % len(building_machines) - for my_node in building_machines: - if my_node['uuid'] in to_ignore: continue - p_nodes = [x for x in provider_nodes if x.uuid == my_node['uuid']] - if len(p_nodes) != 1: - print "Incorrect number of nodes (%s) from provider matching UUID %s" % (len(p_nodes), my_node['uuid']) - to_ignore.append(my_node) - else: - p_node = p_nodes[0] - if (p_node.public_ips and p_node.state == NodeState.RUNNING): - print "Node %s is running, testing ssh" % my_node['id'] - try: - if check_ssh(p_node.public_ip[0]): - print "Node %s is ready" % my_node['id'] - db.setMachineState(my_node['uuid'], vmdatabase.READY) - except Exception, e: - traceback.print_exc() - print "Abandoning node %s due to ssh failure" % (my_node['id']) - db.setMachineState(my_node['uuid'], vmdatabase.ERROR) - error = True - elif (p_node.public_ips and p_node.state in - [NodeState.UNKNOWN, - NodeState.REBOOTING, - NodeState.TERMINATED]): - count = error_counts.get(my_node['id'], 0) - count += 1 - error_counts[my_node['id']] = count - print "Node %s is in error %s (%s/5)" % (my_node['id'], - p_node.state, - count) - if count >= 5: - print "Abandoning node %s due to too many errors" % (my_node['id']) - db.setMachineState(my_node['uuid'], vmdatabase.ERROR) - error = True - else: - if time.time()-my_node['state_time'] >= ABANDON_TIMEOUT: - print "Abandoning node %s due to timeout" % (my_node['id']) - db.setMachineState(my_node['uuid'], vmdatabase.ERROR) - error = True + for machine in building_machines: + try: + check_machine(client, machine, error_counts) + except: + traceback.print_exc() + print "Abandoning machine %s" % machine.id + machine.state = vmdatabase.ERROR + error = True + db.commit() + time.sleep(3) -if error: - sys.exit(1) + + if error: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/devstack-vm-reap.py b/devstack-vm-reap.py index 4c8863ea..a2b697c5 100755 --- a/devstack-vm-reap.py +++ b/devstack-vm-reap.py @@ -2,7 +2,7 @@ # Remove old devstack VMs that have been given to developers. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,56 +18,142 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, sys, time +import os +import sys +import time import getopt - -from libcloud.compute.base import NodeImage, NodeSize, NodeLocation -from libcloud.compute.types import Provider -from libcloud.compute.providers import get_driver +import traceback import vmdatabase +import utils +import novaclient -CLOUD_SERVERS_DRIVER = os.environ.get('CLOUD_SERVERS_DRIVER','rackspace') -CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME'] -CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY'] -MACHINE_LIFETIME = 24*60*60 # Amount of time after being used +PROVIDER_NAME = sys.argv[1] +MACHINE_LIFETIME = 24 * 60 * 60 # Amount of time after being used -db = vmdatabase.VMDatabase() - -if '--all' in sys.argv: +if '--all-servers' in sys.argv: print "Reaping all known machines" - REAP_ALL = True + REAP_ALL_SERVERS = True else: - REAP_ALL = False + REAP_ALL_SERVERS = False -print 'Known machines (start):' -for machine in db.getMachines(): - print machine +if '--all-images' in sys.argv: + print "Reaping all known images" + REAP_ALL_IMAGES = True +else: + REAP_ALL_IMAGES = False -if CLOUD_SERVERS_DRIVER == 'rackspace': - Driver = get_driver(Provider.RACKSPACE) - conn = Driver(CLOUD_SERVERS_USERNAME, CLOUD_SERVERS_API_KEY) -def delete(machine): - node = [n for n in conn.list_nodes() if n.id==str(machine['id'])] - if not node: - print ' Machine id %s not found' % machine['id'] - db.delMachine(machine['uuid']) - return - node = node[0] - node.destroy() - db.delMachine(machine['uuid']) +def delete_machine(client, machine): + try: + server = client.servers.get(machine.external_id) + except novaclient.exceptions.NotFound: + print ' Machine id %s not found' % machine.external_id + server = None -now = time.time() -for machine in db.getMachines(): - # Normally, reap machines that have sat in their current state - # for 24 hours, unless that state is READY. - if REAP_ALL or (machine['state']!=vmdatabase.READY and - now-machine['state_time'] > MACHINE_LIFETIME): - print 'Deleting', machine['name'] - delete(machine) - -print -print 'Known machines (end):' -for machine in db.getMachines(): - print machine + if server: + utils.delete_server(server) + + machine.delete() + + +def delete_image(client, image): + try: + server = client.servers.get(image.server_external_id) + except novaclient.exceptions.NotFound: + print ' Image server id %s not found' % image.server_external_id + server = None + + if server: + utils.delete_server(server) + + try: + remote_image = client.images.get(image.external_id) + except novaclient.exceptions.NotFound: + print ' Image id %s not found' % image.external_id + remote_image = None + + if remote_image: + remote_image.delete() + + image.delete() + + +def main(): + db = vmdatabase.VMDatabase() + + print 'Known machines (start):' + db.print_state() + + provider = db.getProvider(PROVIDER_NAME) + print "Working with provider %s" % provider.name + + client = utils.get_client(provider) + + flavor = utils.get_flavor(client, 1024) + print "Found flavor", flavor + + error = False + now = time.time() + for machine in provider.machines: + # Normally, reap machines that have sat in their current state + # for 24 hours, unless that state is READY. + if REAP_ALL_SERVERS or (machine.state != vmdatabase.READY and + now - machine.state_time > MACHINE_LIFETIME): + print 'Deleting machine', machine.name + try: + delete_machine(client, machine) + except: + error = True + traceback.print_exc() + + provider_min_ready = 0 + for base_image in provider.base_images: + provider_min_ready += base_image.min_ready + for snap_image in base_image.snapshot_images: + # Normally, reap images that have sat in their current state + # for 24 hours, unless the image is the current snapshot + if REAP_ALL_IMAGES or (snap_image != base_image.current_snapshot and + now - snap_image.state_time > MACHINE_LIFETIME): + print 'Deleting image', snap_image.name + try: + delete_image(client, snap_image) + except: + error = True + traceback.print_exc() + + # Make sure the provider has enough headroom for the min_ready + # of all base images, deleting used serverss if needed. + overcommitment = ((len(provider.machines) - + len(provider.ready_machines) + provider_min_ready) - + provider.max_servers) + + while overcommitment > 0: + print 'Overcommitted by %s machines' % overcommitment + last_overcommitment = overcommitment + for machine in provider.machines: + if machine.state == vmdatabase.READY: + continue + if machine.state == vmdatabase.BUILDING: + continue + print 'Deleting machine', machine.name + try: + delete_machine(client, machine) + overcommitment -= 1 + except: + error = True + traceback.print_exc() + if overcommitment == last_overcommitment: + raise Exception("Unable to reduce overcommitment") + last_overcommitment = overcommitment + + print + print 'Known machines (end):' + db.print_state() + + if error: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/devstack-vm-update-image.py b/devstack-vm-update-image.py index 83cb54be..6abbe845 100755 --- a/devstack-vm-update-image.py +++ b/devstack-vm-update-image.py @@ -2,7 +2,7 @@ # Update the base image that is used for devstack VMs. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,31 +18,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -from libcloud.compute.base import NodeImage, NodeSize, NodeLocation -from libcloud.compute.types import Provider -from libcloud.compute.providers import get_driver -from libcloud.compute.deployment import MultiStepDeployment, ScriptDeployment, SSHKeyDeployment -import libcloud + import sys import os import commands import time -import paramiko import subprocess +import traceback +import socket + +import vmdatabase +import utils +from sshclient import SSHClient -CLOUD_SERVERS_DRIVER = os.environ.get('CLOUD_SERVERS_DRIVER','rackspace') -CLOUD_SERVERS_USERNAME = os.environ['CLOUD_SERVERS_USERNAME'] -CLOUD_SERVERS_API_KEY = os.environ['CLOUD_SERVERS_API_KEY'] WORKSPACE = os.environ['WORKSPACE'] +DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '') DEVSTACK = os.path.join(WORKSPACE, 'devstack') -SERVER_NAME = os.environ.get('SERVER_NAME', - 'devstack-oneiric.template.openstack.org') -IMAGE_NAME = os.environ.get('IMAGE_NAME', 'devstack-oneiric') -DISTRIBUTION = 'oneiric' +PROVIDER_NAME = sys.argv[1] + PROJECTS = ['openstack/nova', - 'openstack/glance', - 'openstack/keystone', - 'openstack/horizon', + 'openstack/glance', + 'openstack/keystone', + 'openstack/horizon', 'openstack/python-novaclient', 'openstack/python-keystoneclient', 'openstack/python-quantumclient', @@ -60,6 +57,7 @@ def run_local(cmd, status=False, cwd='.', env={}): return (p.returncode, out.strip()) return out.strip() + def git_branches(): branches = [] for branch in run_local(['git', 'branch'], cwd=DEVSTACK).split("\n"): @@ -68,9 +66,10 @@ def git_branches(): branches.append(branch.strip()) return branches -def tokenize(fn, tokens, comment=None): + +def tokenize(fn, tokens, distribution, comment=None): for line in open(fn): - if 'dist:' in line and ('dist:%s'%DISTRIBUTION not in line): + if 'dist:' in line and ('dist:%s' % distribution not in line): continue if comment and comment in line: line = line[:line.rfind(comment)] @@ -78,138 +77,198 @@ def tokenize(fn, tokens, comment=None): if line and line not in tokens: tokens.append(line) -BRANCHES = [] -for branch in git_branches(): - branch_data = {'name': branch} - print 'Branch: ', branch - run_local(['git', 'checkout', branch], cwd=DEVSTACK) - run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) - pips = [] - pipdir = os.path.join(DEVSTACK, 'files', 'pips') - for fn in os.listdir(pipdir): - fn = os.path.join(pipdir, fn) - tokenize(fn, pips) - branch_data['pips'] = pips +def local_prep(distribution): + branches = [] + for branch in git_branches(): + branch_data = {'name': branch} + print 'Branch: ', branch + run_local(['git', 'checkout', branch], cwd=DEVSTACK) + run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) - debs = [] - debdir = os.path.join(DEVSTACK, 'files', 'apts') - for fn in os.listdir(debdir): - fn = os.path.join(debdir, fn) - tokenize(fn, debs, comment='#') - branch_data['debs'] = debs + pips = [] + pipdir = os.path.join(DEVSTACK, 'files', 'pips') + for fn in os.listdir(pipdir): + fn = os.path.join(pipdir, fn) + tokenize(fn, pips, distribution) + branch_data['pips'] = pips - images = [] - for line in open(os.path.join(DEVSTACK, 'stackrc')): - if line.startswith('IMAGE_URLS'): - if '#' in line: - line = line[:line.rfind('#')] - value = line.split('=', 1)[1].strip() - if value[0]==value[-1]=='"': - value=value[1:-1] - images += [x.strip() for x in value.split(',')] - branch_data['images'] = images - BRANCHES.append(branch_data) + debs = [] + debdir = os.path.join(DEVSTACK, 'files', 'apts') + for fn in os.listdir(debdir): + fn = os.path.join(debdir, fn) + tokenize(fn, debs, distribution, comment='#') + branch_data['debs'] = debs -if CLOUD_SERVERS_DRIVER == 'rackspace': - Driver = get_driver(Provider.RACKSPACE) - conn = Driver(CLOUD_SERVERS_USERNAME, CLOUD_SERVERS_API_KEY) + images = [] + for line in open(os.path.join(DEVSTACK, 'stackrc')): + line = line.strip() + if line.startswith('IMAGE_URLS'): + if '#' in line: + line = line[:line.rfind('#')] + if line.endswith(';;'): + line = line[:-2] + value = line.split('=', 1)[1].strip() + if value[0] == value[-1] == '"': + value = value[1:-1] + images += [x.strip() for x in value.split(',')] + branch_data['images'] = images + branches.append(branch_data) + return branches + + +def bootstrap_server(provider, server, admin_pass, key): + client = server.manager.api + if 'os-floating-ips' in utils.get_extensions(client): + utils.add_public_ip(server) + ip = utils.get_public_ip(server) + if not ip: + raise Exception("Unable to find public ip of server") + + ssh_kwargs = {} + if key: + ssh_kwargs['pkey'] = key + else: + ssh_kwargs['password'] = admin_pass - print "Searching for %s server" % SERVER_NAME - node = [n for n in conn.list_nodes() if n.name==SERVER_NAME][0] + for username in ['root', 'ubuntu']: + client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) + if client: break - print "Searching for %s image" % IMAGE_NAME - old_images = [img for img in conn.list_images() - if img.name.startswith(IMAGE_NAME)] -else: - raise Exception ("Driver not supported") + if not client: + raise Exception("Unable to log in via SSH") -ip = node.public_ip[0] -client = paramiko.SSHClient() -client.load_system_host_keys() -client.set_missing_host_key_policy(paramiko.WarningPolicy()) -client.connect(ip) + # hpcloud can't reliably set the hostname + client.ssh("set hostname", "sudo hostname %s" % server.name) + client.ssh("update apt cache", "sudo apt-get update") + client.ssh("upgrading system packages", "sudo apt-get -y --force-yes upgrade") + client.ssh("install git and puppet", + "sudo apt-get install -y --force-yes git puppet") + client.ssh("clone puppret repo", + "sudo git clone https://review.openstack.org/p/openstack/openstack-ci-puppet.git /root/openstack-ci-puppet") + client.ssh("run puppet", + "sudo puppet apply --modulepath=/root/openstack-ci-puppet/modules /root/openstack-ci-puppet/manifests/site.pp") -def ssh(action, x): - stdin, stdout, stderr = client.exec_command(x) - print x - output = '' - for x in stdout: - output += x - sys.stdout.write(x) - ret = stdout.channel.recv_exit_status() - print stderr.read() - if ret: - raise Exception("Unable to %s" % action) - return output +def configure_server(server, branches): + client = SSHClient(utils.get_public_ip(server), 'jenkins') + client.ssh('make file cache directory', 'mkdir -p ~/cache/files') + client.ssh('make pip cache directory', 'mkdir -p ~/cache/pip') + client.ssh('install build-essential', 'sudo apt-get install -y --force-yes build-essential python-dev') -def scp(source, dest): - print 'copy', source, dest - ftp = client.open_sftp() - ftp.put(source, dest) - ftp.close() + for branch_data in branches: + client.ssh('cache debs for branch %s' % branch_data['name'], + 'sudo apt-get -y -d install %s' % ' '.join(branch_data['debs'])) + venv = client.ssh('get temp dir for venv', 'mktemp -d').strip() + client.ssh('create venv', 'virtualenv --no-site-packages %s' % venv) + client.ssh('cache pips for branch %s' % branch_data['name'], + 'source %s/bin/activate && PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' % + (venv, ' '.join(branch_data['pips']))) + client.ssh('remove venv', 'rm -fr %s' % venv) + for url in branch_data['images']: + fname = url.split('/')[-1] + try: + client.ssh('check for %s' % fname, 'ls ~/cache/files/%s' % fname) + except: + client.ssh('download image %s' % fname, + 'wget -c %s -O ~/cache/files/%s' % (url, fname)) -ssh('make file cache directory', 'mkdir -p ~/cache/files') -ssh('make pip cache directory', 'mkdir -p ~/cache/pip') -ssh('update package list', 'sudo apt-get update') -ssh('upgrade server', 'sudo apt-get -y dist-upgrade') -ssh('run puppet', 'sudo bash -c "cd /root/openstack-ci-puppet && /usr/bin/git pull -q && /var/lib/gems/1.8/bin/puppet apply -l /tmp/manifest.log --modulepath=/root/openstack-ci-puppet/modules manifests/site.pp"') + client.ssh('clear workspace', 'rm -rf ~/workspace') + client.ssh('make workspace', 'mkdir -p ~/workspace') + for project in PROJECTS: + sp = project.split('/')[0] + client.ssh('clone %s' % project, + 'cd ~/workspace && git clone https://review.openstack.org/p/%s' % project) -for branch_data in BRANCHES: - ssh('cache debs for branch %s'%branch_data['name'], - 'sudo apt-get -y -d install %s' % ' '.join(branch_data['debs'])) - venv = ssh('get temp dir for venv', 'mktemp -d').strip() - ssh('create venv', 'virtualenv --no-site-packages %s' % venv) - ssh('cache pips for branch %s'%branch_data['name'], - 'source %s/bin/activate && PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' % (venv, ' '.join(branch_data['pips']))) - ssh('remove venv', 'rm -fr %s'%venv) - for url in branch_data['images']: - fname = url.split('/')[-1] + +def snapshot_server(client, server, name): + print 'Saving image' + if hasattr(client.images, 'create'): #v1.0 + image = client.images.create(server, name) + else: + # TODO: fix novaclient so it returns an image here + # image = server.create_image(name) + uuid = server.manager.create_image(server, name) + image = client.images.get(uuid) + image = utils.wait_for_resource(image) + return image + +def build_image(provider, client, base_image, image, flavor, name, branches, timestamp): + print "Building image %s" % name + + create_kwargs = dict(image=image, flavor=flavor, name=name) + + key = None + key_name = '%sdevstack-%i' % (DEVSTACK_GATE_PREFIX, time.time()) + if 'os-keypairs' in utils.get_extensions(client): + print "Adding keypair" + key, kp = utils.add_keypair(client, key_name) + create_kwargs['key_name'] = key_name + + server = client.servers.create(**create_kwargs) + snap_image = base_image.newSnapshotImage(name=name, + version=timestamp, + external_id=None, + server_external_id=server.id) + admin_pass = server.adminPass + try: + server = utils.wait_for_resource(server) + bootstrap_server(provider, server, admin_pass, key) + configure_server(server, branches) + remote_snap_image = snapshot_server(client, server, name) + snap_image.external_id = remote_snap_image.id + snap_image.state = vmdatabase.READY + # We made the snapshot, try deleting the server, but it's okay + # if we fail. The reap script will find it and try again. try: - ssh('check for %s'%fname, 'ls ~/cache/files/%s'%fname) + utils.delete_server(server) except: - ssh('download image %s'%fname, - 'wget -c %s -O ~/cache/files/%s' % (url, fname)) - -ssh('clear workspace', 'rm -rf ~/workspace') -ssh('make workspace', 'mkdir -p ~/workspace') -for project in PROJECTS: - sp = project.split('/')[0] - ssh('clone %s'%project, - 'cd ~/workspace && git clone https://review.openstack.org/p/%s'%project) - -# TODO: remove after mysql/rabbitmq are removed from image -try: - ssh('stop mysql', 'sudo /etc/init.d/mysql stop') -except: - pass -try: - ssh('stop rabbitmq', 'sudo /etc/init.d/rabbitmq-server stop') -except: - pass - -IMAGE_NAME = IMAGE_NAME+'-'+str(int(time.time())) - -print 'Saving image' -image = conn.ex_save_image(node=node, name=IMAGE_NAME) - -last_extra = None -okay = False -while True: - image = [img for img in conn.list_images(ex_only_active=False) - if img.name==IMAGE_NAME][0] - if image.extra != last_extra: - print image.extra['status'], image.extra['progress'] - if image.extra['status'] == 'ACTIVE': - okay = True - break - last_extra = image.extra - time.sleep(2) - -if okay: - for image in old_images: - print 'Deleting image', image + print "Exception encountered deleting server:" + traceback.print_exc() + except Exception, real_error: + # Something went wrong, try our best to mark the server in error + # then delete the server, then delete the db record for it. + # If any of this fails, the reap script should catch it. But + # having correct info in the DB will help it do its job faster. try: - conn.ex_delete_image(image) - except Exception, e: - print e + snap_image.state = vmdatabase.ERROR + try: + utils.delete_server(server) + snap_image.delete() + except Exception, delete_error: + print "Exception encountered deleting server:" + traceback.print_exc() + except Execption, database_error: + print "Exception encountered marking server in error:" + traceback.print_exc() + # Raise the important exception that started this + raise + + +def main(): + db = vmdatabase.VMDatabase() + provider = db.getProvider(PROVIDER_NAME) + print "Working with provider %s" % provider.name + client = utils.get_client(provider) + + for base_image in provider.base_images: + if base_image.min_ready <= 0: + continue + print "Working on base image %s" % base_image.name + + flavor = utils.get_flavor(client, base_image.min_ram) + print "Found flavor", flavor + + branches = local_prep(base_image.name) + + remote_base_image = client.images.find(name=base_image.external_id) + timestamp = int(time.time()) + remote_snap_image_name = ('%sdevstack-%s-%s.template.openstack.org' % + (DEVSTACK_GATE_PREFIX, base_image.name, str(timestamp))) + remote_snap_image = build_image(provider, client, base_image, + remote_base_image, flavor, + remote_snap_image_name, + branches, timestamp) + + +if __name__ == '__main__': + main() diff --git a/devstack-vm-update-image.sh b/devstack-vm-update-image.sh index 73144e76..39a76557 100755 --- a/devstack-vm-update-image.sh +++ b/devstack-vm-update-image.sh @@ -2,7 +2,7 @@ # Update the VM used in devstack deployments. -# Copyright (C) 2011 OpenStack LLC. +# Copyright (C) 2011-2012 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -CI_SCRIPT_DIR=$(cd $(dirname "$0") && pwd) +GATE_SCRIPT_DIR=$(cd $(dirname "$0") && pwd) cd $WORKSPACE if [[ ! -e devstack ]]; then @@ -29,4 +29,5 @@ git remote update git remote prune origin cd $WORKSPACE -$CI_SCRIPT_DIR/devstack-vm-update-image.py +$GATE_SCRIPT_DIR/devstack-vm-update-image.py $1 + diff --git a/sshclient.py b/sshclient.py new file mode 100644 index 00000000..421f4eaf --- /dev/null +++ b/sshclient.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Update the base image that is used for devstack VMs. + +# Copyright (C) 2011-2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import paramiko +import sys + +class SSHClient(object): + def __init__(self, ip, username, password=None, pkey=None): + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + client.connect(ip, username=username, password=password, pkey=pkey) + self.client = client + + def ssh(self, action, command): + stdin, stdout, stderr = self.client.exec_command(command) + print command + output = '' + for x in stdout: + output += x + sys.stdout.write(x) + ret = stdout.channel.recv_exit_status() + print stderr.read() + if ret: + raise Exception("Unable to %s" % action) + return output + + def scp(self, source, dest): + print 'copy', source, dest + ftp = self.client.open_sftp() + ftp.put(source, dest) + ftp.close() + + diff --git a/tests/test_vmdatabase.py b/tests/test_vmdatabase.py new file mode 100644 index 00000000..e46ce1dc --- /dev/null +++ b/tests/test_vmdatabase.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python + +# Update the base image that is used for devstack VMs. + +# Copyright (C) 2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import vmdatabase + +class testVMDatabase(unittest.TestCase): + + def setUp(self): + self.db = vmdatabase.VMDatabase(':memory:') + + def test_add_provider(self): + provider = vmdatabase.Provider(name='rackspace', driver='rackspace', + username='testuser', api_key='testapikey', + giftable=False) + self.db.session.add(provider) + self.db.commit() + provider = vmdatabase.Provider(name='hpcloud', driver='openstack', + username='testuser', api_key='testapikey', + giftable=True) + self.db.session.add(provider) + self.db.commit() + + def test_add_base_image(self): + self.test_add_provider() + + provider = self.db.getProvider('rackspace') + base_image1 = provider.newBaseImage('oneiric', 1) + base_image2 = provider.newBaseImage('precise', 2) + + provider = self.db.getProvider('hpcloud') + base_image1 = provider.newBaseImage('oneiric', 1) + base_image2 = provider.newBaseImage('precise', 2) + + def test_add_snap_image(self): + self.test_add_base_image() + provider = self.db.getProvider('rackspace') + base_image1 = provider.getBaseImage('oneiric') + base_image2 = provider.getBaseImage('precise') + snapshot_image1 = base_image1.newSnapshotImage('oneiric-1331683549', 1331683549, 201, 301) + snapshot_image2 = base_image2.newSnapshotImage('precise-1331683549', 1331683549, 202, 301) + + hp_provider = self.db.getProvider('hpcloud') + hp_base_image1 = hp_provider.getBaseImage('oneiric') + hp_base_image2 = hp_provider.getBaseImage('precise') + hp_snapshot_image1 = hp_base_image1.newSnapshotImage('oneiric-1331683549', 1331929410, 211, 311) + hp_snapshot_image2 = hp_base_image2.newSnapshotImage('precise-1331683549', 1331929410, 212, 311) + + self.db.print_state() + assert(not base_image1.current_snapshot) + assert(not base_image2.current_snapshot) + + snapshot_image1.state=vmdatabase.READY + assert(base_image1.current_snapshot) + assert(not base_image2.current_snapshot) + assert(snapshot_image1 == base_image1.current_snapshot) + + snapshot_image2.state=vmdatabase.READY + assert(base_image1.current_snapshot) + assert(base_image2.current_snapshot) + assert(snapshot_image1 == base_image1.current_snapshot) + assert(snapshot_image2 == base_image2.current_snapshot) + + snapshot_image2_latest = base_image2.newSnapshotImage('precise-1331683550', + 1331683550, 203, 303) + assert(base_image1.current_snapshot) + assert(base_image2.current_snapshot) + assert(snapshot_image1 == base_image1.current_snapshot) + assert(snapshot_image2 == base_image2.current_snapshot) + + snapshot_image2_latest.state=vmdatabase.READY + assert(base_image1.current_snapshot) + assert(base_image2.current_snapshot) + assert(snapshot_image1 == base_image1.current_snapshot) + assert(snapshot_image2_latest == base_image2.current_snapshot) + + def test_add_machine(self): + self.test_add_snap_image() + provider = self.db.getProvider('rackspace') + base_image1 = provider.getBaseImage('oneiric') + base_image2 = provider.getBaseImage('precise') + snapshot_image1 = base_image1.current_snapshot + snapshot_image2 = base_image2.current_snapshot + assert(len(provider.machines) == 0) + assert(len(provider.ready_machines) == 0) + assert(len(provider.building_machines) == 0) + assert(len(base_image1.machines) == 0) + assert(len(base_image1.ready_machines) == 0) + assert(len(base_image1.building_machines) == 0) + assert(len(base_image2.machines) == 0) + assert(len(base_image2.ready_machines) == 0) + assert(len(base_image2.building_machines) == 0) + + machine1 = base_image1.newMachine('%s-1331683760'%base_image1.name, + '20000021', '1.2.3.4', 'uuid1') + assert(len(provider.machines) == 1) + assert(len(provider.ready_machines) == 0) + assert(len(provider.building_machines) == 1) + assert(len(base_image1.machines) == 1) + assert(len(base_image1.ready_machines) == 0) + assert(len(base_image1.building_machines) == 1) + assert(len(base_image2.machines) == 0) + assert(len(base_image2.ready_machines) == 0) + assert(len(base_image2.building_machines) == 0) + + machine2 = base_image2.newMachine('%s-1331683761'%base_image1.name, + '20000022', '1.2.3.5', 'uuid2') + assert(len(provider.machines) == 2) + assert(len(provider.ready_machines) == 0) + assert(len(provider.building_machines) == 2) + assert(len(base_image1.machines) == 1) + assert(len(base_image1.ready_machines) == 0) + assert(len(base_image1.building_machines) == 1) + assert(len(base_image2.machines) == 1) + assert(len(base_image2.ready_machines) == 0) + assert(len(base_image2.building_machines) == 1) + + machine1.state = vmdatabase.READY + assert(len(provider.machines) == 2) + assert(len(provider.ready_machines) == 1) + assert(len(provider.building_machines) == 1) + assert(len(base_image1.machines) == 1) + assert(len(base_image1.ready_machines) == 1) + assert(len(base_image1.building_machines) == 0) + assert(len(base_image2.machines) == 1) + assert(len(base_image2.ready_machines) == 0) + assert(len(base_image2.building_machines) == 1) + + machine2.state = vmdatabase.ERROR + assert(len(provider.machines) == 2) + assert(len(provider.ready_machines) == 1) + assert(len(provider.building_machines) == 0) + assert(len(base_image1.machines) == 1) + assert(len(base_image1.ready_machines) == 1) + assert(len(base_image1.building_machines) == 0) + assert(len(base_image2.machines) == 1) + assert(len(base_image2.ready_machines) == 0) + assert(len(base_image2.building_machines) == 0) + + machine2.state = vmdatabase.READY + assert(len(provider.machines) == 2) + assert(len(provider.ready_machines) == 2) + assert(len(provider.building_machines) == 0) + assert(len(base_image1.machines) == 1) + assert(len(base_image1.ready_machines) == 1) + assert(len(base_image1.building_machines) == 0) + assert(len(base_image2.machines) == 1) + assert(len(base_image2.ready_machines) == 1) + assert(len(base_image2.building_machines) == 0) + + hp_provider = self.db.getProvider('hpcloud') + hp_base_image1 = hp_provider.getBaseImage('oneiric') + hp_base_image2 = hp_provider.getBaseImage('precise') + hp_snapshot_image1 = hp_base_image1.current_snapshot + hp_snapshot_image2 = hp_base_image2.current_snapshot + hp_machine1 = hp_base_image1.newMachine('%s-1331683551'%hp_base_image1.name, + '21000021', '2.2.3.4', 'hpuuid1') + hp_machine2 = hp_base_image2.newMachine('%s-1331683552'%hp_base_image2.name, + '21000022', '2.2.3.5', 'hpuuid2') + hp_machine1.state = vmdatabase.READY + hp_machine2.state = vmdatabase.READY + + return (machine1, machine2, hp_machine1, hp_machine2) + + def test_get_machine(self): + (machine1, machine2, hp_machine1, hp_machine2) = self.test_add_machine() + # order should be rs1, hp1 for oneiric, hp2, rs1 for precise + hp_machine2.state_time = machine1.state_time-60 + self.db.commit() + + self.db.print_state() + + rs_provider = self.db.getProvider('rackspace') + hp_provider = self.db.getProvider('hpcloud') + + assert(len(rs_provider.ready_machines)==2) + assert(len(hp_provider.ready_machines)==2) + + machine = self.db.getMachineForUse('oneiric') + print 'got machine', machine.name + assert(len(rs_provider.ready_machines)==1) + assert(len(hp_provider.ready_machines)==2) + assert(machine==machine1) + + machine = self.db.getMachineForUse('oneiric') + print 'got machine', machine.name + assert(len(rs_provider.ready_machines)==1) + assert(len(hp_provider.ready_machines)==1) + assert(machine==hp_machine1) + + machine = self.db.getMachineForUse('precise') + print 'got machine', machine.name + assert(len(rs_provider.ready_machines)==1) + assert(len(hp_provider.ready_machines)==0) + assert(machine==hp_machine2) + + machine = self.db.getMachineForUse('precise') + print 'got machine', machine.name + assert(len(rs_provider.ready_machines)==0) + assert(len(hp_provider.ready_machines)==0) + assert(machine==machine2) + diff --git a/utils.py b/utils.py new file mode 100644 index 00000000..5d1216c7 --- /dev/null +++ b/utils.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python + +# Update the base image that is used for devstack VMs. + +# Copyright (C) 2011-2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import novaclient +from novaclient.v1_1 import client as Client11 +from v1_0 import client as Client10 +import time +import os +import traceback +import paramiko +import socket +from sshclient import SSHClient + + +def iterate_timeout(max_seconds, purpose): + start = time.time() + count = 0 + while (time.time() < start + max_seconds): + count += 1 + yield count + time.sleep(2) + raise Exception("Timeout waiting for %s" % purpose) + + +def get_client(provider): + args = [provider.nova_username, provider.nova_api_key, + provider.nova_project_id, provider.nova_auth_url] + kwargs = {} + if provider.nova_service_type: + kwargs['service_type'] = provider.nova_service_type + if provider.nova_service_name: + kwargs['service_name'] = provider.nova_service_name + if provider.nova_service_region: + kwargs['region_name'] = provider.nova_service_region + if provider.nova_api_version == '1.0': + Client = Client10.Client + elif provider.nova_api_version == '1.1': + Client = Client11.Client + else: + raise Exception("API version not supported") + if provider.nova_rax_auth: + os.environ['NOVA_RAX_AUTH'] = '1' + client = Client(*args, **kwargs) + return client + +extension_cache = {} +def get_extensions(client): + global extension_cache + cache = extension_cache.get(client) + if cache: + return cache + try: + resp, body = client.client.get('/extensions') + extensions = [x['alias'] for x in body['extensions']] + except novaclient.exceptions.NotFound: + extensions = [] + extension_cache[client] = extensions + return extensions + +def get_flavor(client, min_ram): + flavors = [f for f in client.flavors.list() if f.ram >= min_ram] + flavors.sort(lambda a, b: cmp(a.ram, b.ram)) + return flavors[0] + +def get_public_ip(server, version=4): + if 'os-floating-ips' in get_extensions(server.manager.api): + print 'using floating ips' + for addr in server.manager.api.floating_ips.list(): + print 'checking addr', addr + if addr.instance_id == server.id: + print 'found addr', addr + return addr.ip + else: + print 'no floating ips, addresses:' + print server.addresses + for addr in server.addresses.get('public', []): + if type(addr) == type(u''): # Rackspace/openstack 1.0 + return addr + if addr['version'] == version: #Rackspace/openstack 1.1 + return addr['addr'] + return None + +def add_public_ip(server): + ip = server.manager.api.floating_ips.create() + print "created floating ip", ip + server.add_floating_ip(ip) + for count in iterate_timeout(600, "ip to be added"): + try: + newip = ip.manager.get(ip.id) + except: + print "Unable to get ip details, will retry" + traceback.print_exc() + continue + + if newip.instance_id == server.id: + print 'ip has been added' + return + +def add_keypair(client, name): + key = paramiko.RSAKey.generate(2048) + public_key = key.get_name() + ' ' + key.get_base64() + kp = client.keypairs.create(name, public_key) + return key, kp + +def wait_for_resource(wait_resource): + last_progress = None + last_status = None + # It can take a _very_ long time for Rackspace 1.0 to save an image + for count in iterate_timeout(21600, "waiting for %s" % wait_resource): + try: + resource = wait_resource.manager.get(wait_resource.id) + except: + print "Unable to list resources, will retry" + traceback.print_exc() + continue + + # In Rackspace v1.0, there is no progress attribute while queued + if hasattr(resource, 'progress'): + if last_progress != resource.progress or last_status != resource.status: + print resource.status, resource.progress + last_progress = resource.progress + elif last_status != resource.status: + print resource.status + last_status = resource.status + if resource.status == 'ACTIVE': + return resource + +def ssh_connect(ip, username, connect_kwargs={}, timeout=60): + # HPcloud may return errno 111 for about 30 seconds after adding the IP + for count in iterate_timeout(timeout, "ssh access"): + try: + client = SSHClient(ip, username, **connect_kwargs) + break + except socket.error, e: + print "While testing ssh access:", e + + out = client.ssh("test ssh access", "echo access okay") + if "access okay" in out: + return client + return None + +def delete_server(server): + try: + if 'os-floating-ips' in get_extensions(server.manager.api): + for addr in server.manager.api.floating_ips.list(): + if addr.instance_id == server.id: + server.remove_floating_ip(addr) + addr.delete() + except: + print "Unable to remove floating IP" + traceback.print_exc() + + try: + if 'os-keypairs' in get_extensions(server.manager.api): + for kp in server.manager.api.keypairs.list(): + if kp.name == server.key_name: + kp.delete() + except: + print "Unable to delete keypair" + traceback.print_exc() + + print "Deleting server", server.id + server.delete() diff --git a/v1_0/__init__.py b/v1_0/__init__.py new file mode 100644 index 00000000..32c1be5f --- /dev/null +++ b/v1_0/__init__.py @@ -0,0 +1 @@ +from client import Client diff --git a/v1_0/accounts.py b/v1_0/accounts.py new file mode 100644 index 00000000..4a455852 --- /dev/null +++ b/v1_0/accounts.py @@ -0,0 +1,18 @@ +from novaclient import base +import base as local_base + + +class Account(base.Resource): + pass + + +class AccountManager(local_base.BootingManagerWithFind): + resource_class = Account + + def create_instance_for(self, account_id, name, image, flavor, + ipgroup=None, meta=None, files=None, zone_blob=None, + reservation_id=None): + resource_url = "/accounts/%s/create_instance" % account_id + return self._boot(resource_url, "server", name, image, flavor, + ipgroup=ipgroup, meta=meta, files=files, + zone_blob=zone_blob, reservation_id=reservation_id) diff --git a/v1_0/backup_schedules.py b/v1_0/backup_schedules.py new file mode 100644 index 00000000..2d8aea82 --- /dev/null +++ b/v1_0/backup_schedules.py @@ -0,0 +1,109 @@ +# Copyright 2010 Jacob Kaplan-Moss +""" +Backup Schedule interface. +""" + +from novaclient import base + + +BACKUP_WEEKLY_DISABLED = 'DISABLED' +BACKUP_WEEKLY_SUNDAY = 'SUNDAY' +BACKUP_WEEKLY_MONDAY = 'MONDAY' +BACKUP_WEEKLY_TUESDAY = 'TUESDAY' +BACKUP_WEEKLY_WEDNESDAY = 'WEDNESDAY' +BACKUP_WEEKLY_THURSDAY = 'THURSDAY' +BACKUP_WEEKLY_FRIDAY = 'FRIDAY' +BACKUP_WEEKLY_SATURDAY = 'SATURDAY' + +BACKUP_DAILY_DISABLED = 'DISABLED' +BACKUP_DAILY_H_0000_0200 = 'H_0000_0200' +BACKUP_DAILY_H_0200_0400 = 'H_0200_0400' +BACKUP_DAILY_H_0400_0600 = 'H_0400_0600' +BACKUP_DAILY_H_0600_0800 = 'H_0600_0800' +BACKUP_DAILY_H_0800_1000 = 'H_0800_1000' +BACKUP_DAILY_H_1000_1200 = 'H_1000_1200' +BACKUP_DAILY_H_1200_1400 = 'H_1200_1400' +BACKUP_DAILY_H_1400_1600 = 'H_1400_1600' +BACKUP_DAILY_H_1600_1800 = 'H_1600_1800' +BACKUP_DAILY_H_1800_2000 = 'H_1800_2000' +BACKUP_DAILY_H_2000_2200 = 'H_2000_2200' +BACKUP_DAILY_H_2200_0000 = 'H_2200_0000' + + +class BackupSchedule(base.Resource): + """ + Represents the daily or weekly backup schedule for some server. + """ + def get(self): + """ + Get this `BackupSchedule` again from the API. + """ + return self.manager.get(server=self.server) + + def delete(self): + """ + Delete (i.e. disable and remove) this scheduled backup. + """ + self.manager.delete(server=self.server) + + def update(self, enabled=True, weekly=BACKUP_WEEKLY_DISABLED, + daily=BACKUP_DAILY_DISABLED): + """ + Update this backup schedule. + + See :meth:`BackupScheduleManager.create` for details. + """ + self.manager.create(self.server, enabled, weekly, daily) + + +class BackupScheduleManager(base.Manager): + """ + Manage server backup schedules. + """ + resource_class = BackupSchedule + + def get(self, server): + """ + Get the current backup schedule for a server. + + :arg server: The server (or its ID). + :rtype: :class:`BackupSchedule` + """ + s = base.getid(server) + schedule = self._get('/servers/%s/backup_schedule' % s, + 'backupSchedule') + schedule.server = server + return schedule + + # Backup schedules use POST for both create and update, so allow both here. + # Unlike the rest of the API, POST here returns no body, so we can't use + # the nice little helper methods. + + def create(self, server, enabled=True, weekly=BACKUP_WEEKLY_DISABLED, + daily=BACKUP_DAILY_DISABLED): + """ + Create or update the backup schedule for the given server. + + :arg server: The server (or its ID). + :arg enabled: boolean; should this schedule be enabled? + :arg weekly: Run a weekly backup on this day + (one of the `BACKUP_WEEKLY_*` constants) + :arg daily: Run a daily backup at this time + (one of the `BACKUP_DAILY_*` constants) + """ + s = base.getid(server) + body = {'backupSchedule': { + 'enabled': enabled, 'weekly': weekly, 'daily': daily + }} + self.api.client.post('/servers/%s/backup_schedule' % s, body=body) + + update = create + + def delete(self, server): + """ + Remove the scheduled backup for `server`. + + :arg server: The server (or its ID). + """ + s = base.getid(server) + self._delete('/servers/%s/backup_schedule' % s) diff --git a/v1_0/base.py b/v1_0/base.py new file mode 100644 index 00000000..f03eb615 --- /dev/null +++ b/v1_0/base.py @@ -0,0 +1,99 @@ +# Copyright 2010 Jacob Kaplan-Moss + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Base utilities to build API operation managers and objects on top of. +""" + +from novaclient import base + + +# Python 2.4 compat +try: + all +except NameError: + def all(iterable): + return True not in (not x for x in iterable) + + +class BootingManagerWithFind(base.ManagerWithFind): + """Like a `ManagerWithFind`, but has the ability to boot servers.""" + def _boot(self, resource_url, response_key, name, image, flavor, + ipgroup=None, meta=None, files=None, zone_blob=None, + reservation_id=None, return_raw=False, min_count=None, + max_count=None): + """ + Create (boot) a new server. + + :param name: Something to name the server. + :param image: The :class:`Image` to boot with. + :param flavor: The :class:`Flavor` to boot onto. + :param ipgroup: An initial :class:`IPGroup` for this server. + :param meta: A dict of arbitrary key/value metadata to store for this + server. A maximum of five entries is allowed, and both + keys and values must be 255 characters or less. + :param files: A dict of files to overrwrite on the server upon boot. + Keys are file names (i.e. ``/etc/passwd``) and values + are the file contents (either as a string or as a + file-like object). A maximum of five entries is allowed, + and each file must be 10k or less. + :param zone_blob: a single (encrypted) string which is used internally + by Nova for routing between Zones. Users cannot populate + this field. + :param reservation_id: a UUID for the set of servers being requested. + :param return_raw: If True, don't try to coearse the result into + a Resource object. + """ + body = {"server": { + "name": name, + "imageId": base.getid(image), + "flavorId": base.getid(flavor), + }} + if ipgroup: + body["server"]["sharedIpGroupId"] = base.getid(ipgroup) + if meta: + body["server"]["metadata"] = meta + if reservation_id: + body["server"]["reservation_id"] = reservation_id + if zone_blob: + body["server"]["blob"] = zone_blob + + if not min_count: + min_count = 1 + if not max_count: + max_count = min_count + body["server"]["min_count"] = min_count + body["server"]["max_count"] = max_count + + # Files are a slight bit tricky. They're passed in a "personality" + # list to the POST. Each item is a dict giving a file name and the + # base64-encoded contents of the file. We want to allow passing + # either an open file *or* some contents as files here. + if files: + personality = body['server']['personality'] = [] + for filepath, file_or_string in files.items(): + if hasattr(file_or_string, 'read'): + data = file_or_string.read() + else: + data = file_or_string + personality.append({ + 'path': filepath, + 'contents': data.encode('base64'), + }) + + return self._create(resource_url, body, response_key, + return_raw=return_raw) diff --git a/v1_0/client.py b/v1_0/client.py new file mode 100644 index 00000000..ae07c5ef --- /dev/null +++ b/v1_0/client.py @@ -0,0 +1,75 @@ +from novaclient import client +import accounts +import backup_schedules +import flavors +import images +import ipgroups +import servers +import zones + + +class Client(object): + """ + Top-level object to access the OpenStack Compute API. + + Create an instance with your creds:: + + >>> client = Client(USERNAME, PASSWORD, PROJECT_ID, AUTH_URL) + + Then call methods on its managers:: + + >>> client.servers.list() + ... + >>> client.flavors.list() + ... + + """ + + def __init__(self, username, api_key, project_id, auth_url=None, + insecure=False, timeout=None, token=None, region_name=None, + endpoint_name=None, extensions=None, service_type=None, + service_name=None, endpoint_type='publicURL'): + + # FIXME(comstud): Rename the api_key argument above when we + # know it's not being used as keyword argument + password = api_key + self.accounts = accounts.AccountManager(self) + self.backup_schedules = backup_schedules.BackupScheduleManager(self) + self.flavors = flavors.FlavorManager(self) + self.images = images.ImageManager(self) + self.ipgroups = ipgroups.IPGroupManager(self) + self.servers = servers.ServerManager(self) + self.zones = zones.ZoneManager(self) + #service_type is unused in v1_0 + #service_name is unused in v1_0 + #endpoint_name is unused in v_10 + #endpoint_type was endpoint_name + + # Add in any extensions... + if extensions: + for (ext_name, ext_manager_class, ext_module) in extensions: + setattr(self, ext_name, ext_manager_class(self)) + + _auth_url = auth_url or 'https://auth.api.rackspacecloud.com/v1.0' + + self.client = client.HTTPClient(username, + password, + project_id, + _auth_url, + insecure=insecure, + timeout=timeout, + proxy_token=token, + region_name=region_name, + endpoint_type=endpoint_type) + + def authenticate(self): + """ + Authenticate against the server. + + Normally this is called automatically when you first access the API, + but you can call this method to force authentication right now. + + Returns on success; raises :exc:`exceptions.Unauthorized` if the + credentials are wrong. + """ + self.client.authenticate() diff --git a/v1_0/flavors.py b/v1_0/flavors.py new file mode 100644 index 00000000..f1b49580 --- /dev/null +++ b/v1_0/flavors.py @@ -0,0 +1,41 @@ +# Copyright 2010 Jacob Kaplan-Moss +""" +Flavor interface. +""" + +from novaclient import base + + +class Flavor(base.Resource): + """ + A flavor is an available hardware configuration for a server. + """ + def __repr__(self): + return "" % self.name + + +class FlavorManager(base.ManagerWithFind): + """ + Manage :class:`Flavor` resources. + """ + resource_class = Flavor + + def list(self, detailed=True): + """ + Get a list of all flavors. + + :rtype: list of :class:`Flavor`. + """ + detail = "" + if detailed: + detail = "/detail" + return self._list("/flavors%s" % detail, "flavors") + + def get(self, flavor): + """ + Get a specific flavor. + + :param flavor: The ID of the :class:`Flavor` to get. + :rtype: :class:`Flavor` + """ + return self._get("/flavors/%s" % base.getid(flavor), "flavor") diff --git a/v1_0/images.py b/v1_0/images.py new file mode 100644 index 00000000..cf063cfe --- /dev/null +++ b/v1_0/images.py @@ -0,0 +1,69 @@ +# Copyright 2010 Jacob Kaplan-Moss +""" +Image interface. +""" + +from novaclient import base +import time + +class Image(base.Resource): + """ + An image is a collection of files used to create or rebuild a server. + """ + def __repr__(self): + return "" % self.name + + def delete(self): + """ + Delete this image. + """ + return self.manager.delete(self) + + +class ImageManager(base.ManagerWithFind): + """ + Manage :class:`Image` resources. + """ + resource_class = Image + + def get(self, image): + """ + Get an image. + + :param image: The ID of the image to get. + :rtype: :class:`Image` + """ + return self._get("/images/%s" % base.getid(image), "image") + + def list(self, detailed=True): + """ + Get a list of all images. + + :rtype: list of :class:`Image` + """ + detail = "" + if detailed: + detail = "/detail" + return self._list("/images%s?cache-busting=%s" % (detail, time.time()), "images") + + def create(self, server, name): + """ + Create a new image by snapshotting a running :class:`Server` + + :param name: An (arbitrary) name for the new image. + :param server: The :class:`Server` (or its ID) to make a snapshot of. + :rtype: :class:`Image` + """ + data = {"image": {"serverId": base.getid(server), "name": name}} + return self._create("/images", data, "image") + + def delete(self, image): + """ + Delete an image. + + It should go without saying that you can't delete an image + that you didn't create. + + :param image: The :class:`Image` (or its ID) to delete. + """ + self._delete("/images/%s" % base.getid(image)) diff --git a/v1_0/ipgroups.py b/v1_0/ipgroups.py new file mode 100644 index 00000000..86cd3cb4 --- /dev/null +++ b/v1_0/ipgroups.py @@ -0,0 +1,64 @@ +# Copyright 2010 Jacob Kaplan-Moss +""" +IP Group interface. +""" + +from novaclient import base + + +class IPGroup(base.Resource): + def __repr__(self): + return "" % self.name + + def delete(self): + """ + Delete this group. + """ + self.manager.delete(self) + + +class IPGroupManager(base.ManagerWithFind): + resource_class = IPGroup + + def list(self, detailed=True): + """ + Get a list of all groups. + + :rtype: list of :class:`IPGroup` + """ + detail = "" + if detailed: + detail = "/detail" + return self._list("/shared_ip_groups%s" % detail, "sharedIpGroups") + + def get(self, group): + """ + Get an IP group. + + :param group: ID of the image to get. + :rtype: :class:`IPGroup` + """ + return self._get("/shared_ip_groups/%s" % base.getid(group), + "sharedIpGroup") + + def create(self, name, server=None): + """ + Create a new :class:`IPGroup` + + :param name: An (arbitrary) name for the new image. + :param server: A :class:`Server` (or its ID) to make a member + of this group. + :rtype: :class:`IPGroup` + """ + data = {"sharedIpGroup": {"name": name}} + if server: + data['sharedIpGroup']['server'] = base.getid(server) + return self._create('/shared_ip_groups', data, "sharedIpGroup") + + def delete(self, group): + """ + Delete a group. + + :param group: The :class:`IPGroup` (or its ID) to delete. + """ + self._delete("/shared_ip_groups/%s" % base.getid(group)) diff --git a/v1_0/servers.py b/v1_0/servers.py new file mode 100644 index 00000000..d4e8f393 --- /dev/null +++ b/v1_0/servers.py @@ -0,0 +1,488 @@ +# Copyright 2010 Jacob Kaplan-Moss + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Server interface. +""" + +import urllib + +from novaclient import base +import base as local_base +import time + +REBOOT_SOFT, REBOOT_HARD = 'SOFT', 'HARD' + + +class Server(base.Resource): + def __repr__(self): + return "" % self.name + + def delete(self): + """ + Delete (i.e. shut down and delete the image) this server. + """ + self.manager.delete(self) + + def update(self, name=None, password=None): + """ + Update the name or the password for this server. + + :param name: Update the server's name. + :param password: Update the root password. + """ + self.manager.update(self, name, password) + + def share_ip(self, ipgroup, address, configure=True): + """ + Share an IP address from the given IP group onto this server. + + :param ipgroup: The :class:`IPGroup` that the given address belongs to. + :param address: The IP address to share. + :param configure: If ``True``, the server will be automatically + configured to use this IP. I don't know why you'd + want this to be ``False``. + """ + self.manager.share_ip(self, ipgroup, address, configure) + + def unshare_ip(self, address): + """ + Stop sharing the given address. + + :param address: The IP address to stop sharing. + """ + self.manager.unshare_ip(self, address) + + def add_fixed_ip(self, network_id): + """ + Add an IP address on a network. + + :param network_id: The ID of the network the IP should be on. + """ + self.manager.add_fixed_ip(self, network_id) + + def remove_fixed_ip(self, address): + """ + Remove an IP address. + + :param address: The IP address to remove. + """ + self.manager.remove_fixed_ip(self, address) + + def reboot(self, type=REBOOT_SOFT): + """ + Reboot the server. + + :param type: either :data:`REBOOT_SOFT` for a software-level reboot, + or `REBOOT_HARD` for a virtual power cycle hard reboot. + """ + self.manager.reboot(self, type) + + def pause(self): + """ + Pause -- Pause the running server. + """ + self.manager.pause(self) + + def unpause(self): + """ + Unpause -- Unpause the paused server. + """ + self.manager.unpause(self) + + def suspend(self): + """ + Suspend -- Suspend the running server. + """ + self.manager.suspend(self) + + def resume(self): + """ + Resume -- Resume the suspended server. + """ + self.manager.resume(self) + + def rescue(self): + """ + Rescue -- Rescue the problematic server. + """ + self.manager.rescue(self) + + def unrescue(self): + """ + Unrescue -- Unrescue the rescued server. + """ + self.manager.unrescue(self) + + def diagnostics(self): + """Diagnostics -- Retrieve server diagnostics.""" + self.manager.diagnostics(self) + + def actions(self): + """Actions -- Retrieve server actions.""" + self.manager.actions(self) + + def rebuild(self, image): + """ + Rebuild -- shut down and then re-image -- this server. + + :param image: the :class:`Image` (or its ID) to re-image with. + """ + self.manager.rebuild(self, image) + + def resize(self, flavor): + """ + Resize the server's resources. + + :param flavor: the :class:`Flavor` (or its ID) to resize to. + + Until a resize event is confirmed with :meth:`confirm_resize`, the old + server will be kept around and you'll be able to roll back to the old + flavor quickly with :meth:`revert_resize`. All resizes are + automatically confirmed after 24 hours. + """ + self.manager.resize(self, flavor) + + def backup(self, image_name, backup_type, rotation): + """ + Create a server backup. + + :param server: The :class:`Server` (or its ID). + :param image_name: The name to assign the newly create image. + :param backup_type: 'daily' or 'weekly' + :param rotation: number of backups of type 'backup_type' to keep + :returns Newly created :class:`Image` object + """ + return self.manager.backup(self, image_name, backup_type, rotation) + + def confirm_resize(self): + """ + Confirm that the resize worked, thus removing the original server. + """ + self.manager.confirm_resize(self) + + def revert_resize(self): + """ + Revert a previous resize, switching back to the old server. + """ + self.manager.revert_resize(self) + + def migrate(self): + """ + Migrate a server to a new host in the same zone. + """ + self.manager.migrate(self) + + @property + def backup_schedule(self): + """ + This server's :class:`BackupSchedule`. + """ + return self.manager.api.backup_schedules.get(self) + + @property + def public_ip(self): + """ + Shortcut to get this server's primary public IP address. + """ + if len(self.addresses['public']) == 0: + return "" + return self.addresses['public'] + + @property + def private_ip(self): + """ + Shortcut to get this server's primary private IP address. + """ + if len(self.addresses['private']) == 0: + return "" + return self.addresses['private'] + + +class ServerManager(local_base.BootingManagerWithFind): + resource_class = Server + + def get(self, server): + """ + Get a server. + + :param server: ID of the :class:`Server` to get. + :rtype: :class:`Server` + """ + return self._get("/servers/%s" % base.getid(server), "server") + + def list(self, detailed=True, search_opts=None): + """ + Get a list of servers. + Optional detailed returns details server info. + Optional reservation_id only returns instances with that + reservation_id. + + :rtype: list of :class:`Server` + """ + if search_opts is None: + search_opts = {} + qparams = {} + # only use values in query string if they are set + for opt, val in search_opts.iteritems(): + if val: + qparams[opt] = val + + query_string = "?%s" % urllib.urlencode(qparams) if qparams else "" + + detail = "" + if detailed: + detail = "/detail" + return self._list("/servers%s%s?cache-busting=%s" % (detail, query_string, time.time()), "servers") + + def create(self, name, image, flavor, ipgroup=None, meta=None, files=None, + zone_blob=None, reservation_id=None, min_count=None, + max_count=None): + """ + Create (boot) a new server. + + :param name: Something to name the server. + :param image: The :class:`Image` to boot with. + :param flavor: The :class:`Flavor` to boot onto. + :param ipgroup: An initial :class:`IPGroup` for this server. + :param meta: A dict of arbitrary key/value metadata to store for this + server. A maximum of five entries is allowed, and both + keys and values must be 255 characters or less. + :param files: A dict of files to overrwrite on the server upon boot. + Keys are file names (i.e. ``/etc/passwd``) and values + are the file contents (either as a string or as a + file-like object). A maximum of five entries is allowed, + and each file must be 10k or less. + :param zone_blob: a single (encrypted) string which is used internally + by Nova for routing between Zones. Users cannot populate + this field. + :param reservation_id: a UUID for the set of servers being requested. + """ + if not min_count: + min_count = 1 + if not max_count: + max_count = min_count + if min_count > max_count: + min_count = max_count + return self._boot("/servers", "server", name, image, flavor, + ipgroup=ipgroup, meta=meta, files=files, + zone_blob=zone_blob, reservation_id=reservation_id, + min_count=min_count, max_count=max_count) + + def update(self, server, name=None, password=None): + """ + Update the name or the password for a server. + + :param server: The :class:`Server` (or its ID) to update. + :param name: Update the server's name. + :param password: Update the root password. + """ + + if name is None and password is None: + return + body = {"server": {}} + if name: + body["server"]["name"] = name + if password: + body["server"]["adminPass"] = password + self._update("/servers/%s" % base.getid(server), body) + + def delete(self, server): + """ + Delete (i.e. shut down and delete the image) this server. + """ + self._delete("/servers/%s" % base.getid(server)) + + def share_ip(self, server, ipgroup, address, configure=True): + """ + Share an IP address from the given IP group onto a server. + + :param server: The :class:`Server` (or its ID) to share onto. + :param ipgroup: The :class:`IPGroup` that the given address belongs to. + :param address: The IP address to share. + :param configure: If ``True``, the server will be automatically + configured to use this IP. I don't know why you'd + want this to be ``False``. + """ + server = base.getid(server) + ipgroup = base.getid(ipgroup) + body = {'shareIp': {'sharedIpGroupId': ipgroup, + 'configureServer': configure}} + self._update("/servers/%s/ips/public/%s" % (server, address), body) + + def unshare_ip(self, server, address): + """ + Stop sharing the given address. + + :param server: The :class:`Server` (or its ID) to share onto. + :param address: The IP address to stop sharing. + """ + server = base.getid(server) + self._delete("/servers/%s/ips/public/%s" % (server, address)) + + def add_fixed_ip(self, server, network_id): + """ + Add an IP address on a network. + + :param server: The :class:`Server` (or its ID) to add an IP to. + :param network_id: The ID of the network the IP should be on. + """ + self._action('addFixedIp', server, {'networkId': network_id}) + + def remove_fixed_ip(self, server, address): + """ + Remove an IP address. + + :param server: The :class:`Server` (or its ID) to add an IP to. + :param address: The IP address to remove. + """ + self._action('removeFixedIp', server, {'address': address}) + + def reboot(self, server, type=REBOOT_SOFT): + """ + Reboot a server. + + :param server: The :class:`Server` (or its ID) to share onto. + :param type: either :data:`REBOOT_SOFT` for a software-level reboot, + or `REBOOT_HARD` for a virtual power cycle hard reboot. + """ + self._action('reboot', server, {'type': type}) + + def rebuild(self, server, image): + """ + Rebuild -- shut down and then re-image -- a server. + + :param server: The :class:`Server` (or its ID) to share onto. + :param image: the :class:`Image` (or its ID) to re-image with. + """ + self._action('rebuild', server, {'imageId': base.getid(image)}) + + def resize(self, server, flavor): + """ + Resize a server's resources. + + :param server: The :class:`Server` (or its ID) to share onto. + :param flavor: the :class:`Flavor` (or its ID) to resize to. + + Until a resize event is confirmed with :meth:`confirm_resize`, the old + server will be kept around and you'll be able to roll back to the old + flavor quickly with :meth:`revert_resize`. All resizes are + automatically confirmed after 24 hours. + """ + self._action('resize', server, {'flavorId': base.getid(flavor)}) + + def backup(self, server, image_name, backup_type, rotation): + """ + Create a server backup. + + :param server: The :class:`Server` (or its ID). + :param image_name: The name to assign the newly create image. + :param backup_type: 'daily' or 'weekly' + :param rotation: number of backups of type 'backup_type' to keep + :returns Newly created :class:`Image` object + """ + if not rotation: + raise Exception("rotation is required for backups") + elif not backup_type: + raise Exception("backup_type required for backups") + elif backup_type not in ("daily", "weekly"): + raise Exception("Invalid backup_type: must be daily or weekly") + + data = { + "name": image_name, + "rotation": rotation, + "backup_type": backup_type, + } + + self._action('createBackup', server, data) + + def pause(self, server): + """ + Pause the server. + """ + self.api.client.post('/servers/%s/pause' % base.getid(server)) + + def unpause(self, server): + """ + Unpause the server. + """ + self.api.client.post('/servers/%s/unpause' % base.getid(server)) + + def suspend(self, server): + """ + Suspend the server. + """ + self.api.client.post('/servers/%s/suspend' % base.getid(server)) + + def resume(self, server): + """ + Resume the server. + """ + self.api.client.post('/servers/%s/resume' % base.getid(server)) + + def rescue(self, server): + """ + Rescue the server. + """ + self.api.client.post('/servers/%s/rescue' % base.getid(server)) + + def unrescue(self, server): + """ + Unrescue the server. + """ + self.api.client.post('/servers/%s/unrescue' % base.getid(server)) + + def diagnostics(self, server): + """Retrieve server diagnostics.""" + return self.api.client.get("/servers/%s/diagnostics" % + base.getid(server)) + + def actions(self, server): + """Retrieve server actions.""" + return self._list("/servers/%s/actions" % base.getid(server), + "actions") + + def confirm_resize(self, server): + """ + Confirm that the resize worked, thus removing the original server. + + :param server: The :class:`Server` (or its ID) to share onto. + """ + self._action('confirmResize', server) + + def revert_resize(self, server): + """ + Revert a previous resize, switching back to the old server. + + :param server: The :class:`Server` (or its ID) to share onto. + """ + self._action('revertResize', server) + + def migrate(self, server): + """ + Migrate a server to a new host in the same zone. + + :param server: The :class:`Server` (or its ID). + """ + self.api.client.post('/servers/%s/migrate' % base.getid(server)) + + def _action(self, action, server, info=None): + """ + Perform a server "action" -- reboot/rebuild/resize/etc. + """ + self.api.client.post('/servers/%s/action' % base.getid(server), + body={action: info}) diff --git a/v1_0/shell.py b/v1_0/shell.py new file mode 100644 index 00000000..cb462727 --- /dev/null +++ b/v1_0/shell.py @@ -0,0 +1,788 @@ +# Copyright 2010 Jacob Kaplan-Moss + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import getpass +import os + +from novaclient import exceptions +from novaclient import utils +import client +import backup_schedules +import servers + + +CLIENT_CLASS = client.Client + +# Choices for flags. +DAY_CHOICES = [getattr(backup_schedules, i).lower() + for i in dir(backup_schedules) + if i.startswith('BACKUP_WEEKLY_')] +HOUR_CHOICES = [getattr(backup_schedules, i).lower() + for i in dir(backup_schedules) + if i.startswith('BACKUP_DAILY_')] + + +# Sentinal for boot --key +AUTO_KEY = object() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('--enable', dest='enabled', default=None, action='store_true', + help='Enable backups.') +@utils.arg('--disable', dest='enabled', action='store_false', + help='Disable backups.') +@utils.arg('--weekly', metavar='', choices=DAY_CHOICES, + help='Schedule a weekly backup for (one of: %s).' % + utils.pretty_choice_list(DAY_CHOICES)) +@utils.arg('--daily', metavar='', choices=HOUR_CHOICES, + help='Schedule a daily backup during (one of: %s).' % + utils.pretty_choice_list(HOUR_CHOICES)) +def do_backup_schedule(cs, args): + """ + Show or edit the backup schedule for a server. + + With no flags, the backup schedule will be shown. If flags are given, + the backup schedule will be modified accordingly. + """ + server = _find_server(cs, args.server) + + # If we have some flags, update the backup + backup = {} + if args.daily: + backup['daily'] = getattr(backup_schedules, 'BACKUP_DAILY_%s' % + args.daily.upper()) + if args.weekly: + backup['weekly'] = getattr(backup_schedules, 'BACKUP_WEEKLY_%s' % + args.weekly.upper()) + if args.enabled is not None: + backup['enabled'] = args.enabled + if backup: + server.backup_schedule.update(**backup) + else: + utils.print_dict(server.backup_schedule._info) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_backup_schedule_delete(cs, args): + """ + Delete the backup schedule for a server. + """ + server = _find_server(cs, args.server) + server.backup_schedule.delete() + + +def _boot(cs, args, reservation_id=None, min_count=None, max_count=None): + """Boot a new server.""" + if min_count is None: + min_count = 1 + if max_count is None: + max_count = min_count + if min_count > max_count: + raise exceptions.CommandError("min_instances should be" + "<= max_instances") + if not min_count or not max_count: + raise exceptions.CommandError("min_instances nor max_instances" + "should be 0") + + flavor = args.flavor or cs.flavors.find(ram=256) + image = args.image or cs.images.find(name="Ubuntu 10.04 LTS "\ + "(lucid)") + + # Map --ipgroup to an ID. + # XXX do this for flavor/image? + if args.ipgroup: + ipgroup = _find_ipgroup(cs, args.ipgroup) + else: + ipgroup = None + + metadata = dict(v.split('=') for v in args.meta) + + files = {} + for f in args.files: + dst, src = f.split('=', 1) + try: + files[dst] = open(src) + except IOError, e: + raise exceptions.CommandError("Can't open '%s': %s" % (src, e)) + + if args.key is AUTO_KEY: + possible_keys = [os.path.join(os.path.expanduser('~'), '.ssh', k) + for k in ('id_dsa.pub', 'id_rsa.pub')] + for k in possible_keys: + if os.path.exists(k): + keyfile = k + break + else: + raise exceptions.CommandError("Couldn't find a key file: tried " + "~/.ssh/id_dsa.pub or ~/.ssh/id_rsa.pub") + elif args.key: + keyfile = args.key + else: + keyfile = None + + if keyfile: + try: + files['/root/.ssh/authorized_keys2'] = open(keyfile) + except IOError, e: + raise exceptions.CommandError("Can't open '%s': %s" % (keyfile, e)) + + return (args.name, image, flavor, ipgroup, metadata, files, + reservation_id, min_count, max_count) + + +@utils.arg('--flavor', + default=None, + type=int, + metavar='', + help="Flavor ID (see 'nova flavors'). "\ + "Defaults to 256MB RAM instance.") +@utils.arg('--image', + default=None, + type=int, + metavar='', + help="Image ID (see 'nova images'). "\ + "Defaults to Ubuntu 10.04 LTS.") +@utils.arg('--ipgroup', + default=None, + metavar='', + help="IP group name or ID (see 'nova ipgroup-list').") +@utils.arg('--meta', + metavar="", + action='append', + default=[], + help="Record arbitrary key/value metadata. "\ + "May be give multiple times.") +@utils.arg('--file', + metavar="", + action='append', + dest='files', + default=[], + help="Store arbitrary files from locally to "\ + "on the new server. You may store up to 5 files.") +@utils.arg('--key', + metavar='', + nargs='?', + const=AUTO_KEY, + help="Key the server with an SSH keypair. "\ + "Looks in ~/.ssh for a key, "\ + "or takes an explicit to one.") +@utils.arg('name', metavar='', help='Name for the new server') +def do_boot(cs, args): + """Boot a new server.""" + name, image, flavor, ipgroup, metadata, files, reservation_id, \ + min_count, max_count = _boot(cs, args) + + server = cs.servers.create(args.name, image, flavor, + ipgroup=ipgroup, + meta=metadata, + files=files, + min_count=min_count, + max_count=max_count) + utils.print_dict(server._info) + + +@utils.arg('--flavor', + default=None, + type=int, + metavar='', + help="Flavor ID (see 'nova flavors'). "\ + "Defaults to 256MB RAM instance.") +@utils.arg('--image', + default=None, + type=int, + metavar='', + help="Image ID (see 'nova images'). "\ + "Defaults to Ubuntu 10.04 LTS.") +@utils.arg('--ipgroup', + default=None, + metavar='', + help="IP group name or ID (see 'nova ipgroup-list').") +@utils.arg('--meta', + metavar="", + action='append', + default=[], + help="Record arbitrary key/value metadata. "\ + "May be give multiple times.") +@utils.arg('--file', + metavar="", + action='append', + dest='files', + default=[], + help="Store arbitrary files from locally to "\ + "on the new server. You may store up to 5 files.") +@utils.arg('--key', + metavar='', + nargs='?', + const=AUTO_KEY, + help="Key the server with an SSH keypair. "\ + "Looks in ~/.ssh for a key, "\ + "or takes an explicit to one.") +@utils.arg('account', metavar='', help='Account to build this'\ + ' server for') +@utils.arg('name', metavar='', help='Name for the new server') +def do_boot_for_account(cs, args): + """Boot a new server in an account.""" + name, image, flavor, ipgroup, metadata, files, reservation_id, \ + min_count, max_count = _boot(cs, args) + + server = cs.accounts.create_instance_for(args.account, args.name, + image, flavor, + ipgroup=ipgroup, + meta=metadata, + files=files) + utils.print_dict(server._info) + + +@utils.arg('--flavor', + default=None, + type=int, + metavar='', + help="Flavor ID (see 'nova flavors'). "\ + "Defaults to 256MB RAM instance.") +@utils.arg('--image', + default=None, + type=int, + metavar='', + help="Image ID (see 'nova images'). "\ + "Defaults to Ubuntu 10.04 LTS.") +@utils.arg('--ipgroup', + default=None, + metavar='', + help="IP group name or ID (see 'nova ipgroup-list').") +@utils.arg('--meta', + metavar="", + action='append', + default=[], + help="Record arbitrary key/value metadata. "\ + "May be give multiple times.") +@utils.arg('--file', + metavar="", + action='append', + dest='files', + default=[], + help="Store arbitrary files from locally to "\ + "on the new server. You may store up to 5 files.") +@utils.arg('--key', + metavar='', + nargs='?', + const=AUTO_KEY, + help="Key the server with an SSH keypair. "\ + "Looks in ~/.ssh for a key, "\ + "or takes an explicit to one.") +@utils.arg('--reservation_id', + default=None, + metavar='', + help="Reservation ID (a UUID). "\ + "If unspecified will be generated by the server.") +@utils.arg('--min_instances', + default=None, + type=int, + metavar='', + help="The minimum number of instances to build. "\ + "Defaults to 1.") +@utils.arg('--max_instances', + default=None, + type=int, + metavar='', + help="The maximum number of instances to build. "\ + "Defaults to 'min_instances' setting.") +@utils.arg('name', metavar='', help='Name for the new server') +def do_zone_boot(cs, args): + """Boot a new server, potentially across Zones.""" + reservation_id = args.reservation_id + min_count = args.min_instances + max_count = args.max_instances + name, image, flavor, ipgroup, metadata, \ + files, reservation_id, min_count, max_count = \ + _boot(cs, args, + reservation_id=reservation_id, + min_count=min_count, + max_count=max_count) + + reservation_id = cs.zones.boot(args.name, image, flavor, + ipgroup=ipgroup, + meta=metadata, + files=files, + reservation_id=reservation_id, + min_count=min_count, + max_count=max_count) + print "Reservation ID=", reservation_id + + +def _translate_flavor_keys(collection): + convert = [('ram', 'memory_mb'), ('disk', 'local_gb')] + for item in collection: + keys = item.__dict__.keys() + for from_key, to_key in convert: + if from_key in keys and to_key not in keys: + setattr(item, to_key, item._info[from_key]) + + +def do_flavor_list(cs, args): + """Print a list of available 'flavors' (sizes of servers).""" + flavors = cs.flavors.list() + _translate_flavor_keys(flavors) + utils.print_list(flavors, [ + 'ID', + 'Name', + 'Memory_MB', + 'Swap', + 'Local_GB', + 'VCPUs', + 'RXTX_Factor']) + + +def do_image_list(cs, args): + """Print a list of available images to boot from.""" + server_list = {} + for server in cs.servers.list(): + server_list[server.id] = server.name + image_list = cs.images.list() + for i in range(len(image_list)): + if hasattr(image_list[i], 'serverId'): + image_list[i].serverId = server_list[image_list[i].serverId] + \ + ' (' + str(image_list[i].serverId) + ')' + utils.print_list(image_list, ['ID', 'Name', 'serverId', 'Status']) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('name', metavar='', help='Name of snapshot.') +def do_image_create(cs, args): + """Create a new image by taking a snapshot of a running server.""" + server = _find_server(cs, args.server) + image = cs.images.create(server, args.name) + utils.print_dict(image._info) + + +@utils.arg('image', metavar='', help='Name or ID of image.') +def do_image_delete(cs, args): + """ + Delete an image. + + It should go without saying, but you can only delete images you + created. + """ + image = _find_image(cs, args.image) + image.delete() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('group', metavar='', help='Name or ID of group.') +@utils.arg('address', metavar='
', help='IP address to share.') +def do_ip_share(cs, args): + """Share an IP address from the given IP group onto a server.""" + server = _find_server(cs, args.server) + group = _find_ipgroup(cs, args.group) + server.share_ip(group, args.address) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('address', metavar='
', + help='Shared IP address to remove from the server.') +def do_ip_unshare(cs, args): + """Stop sharing an given address with a server.""" + server = _find_server(cs, args.server) + server.unshare_ip(args.address) + + +def do_ipgroup_list(cs, args): + """Show IP groups.""" + def pretty_server_list(ipgroup): + return ", ".join(cs.servers.get(id).name + for id in ipgroup.servers) + + utils.print_list(cs.ipgroups.list(), + fields=['ID', 'Name', 'Server List'], + formatters={'Server List': pretty_server_list}) + + +@utils.arg('group', metavar='', help='Name or ID of group.') +def do_ipgroup_show(cs, args): + """Show details about a particular IP group.""" + group = _find_ipgroup(cs, args.group) + utils.print_dict(group._info) + + +@utils.arg('name', metavar='', help='What to name this new group.') +@utils.arg('server', metavar='', nargs='?', + help='Server (name or ID) to make a member of this new group.') +def do_ipgroup_create(cs, args): + """Create a new IP group.""" + if args.server: + server = _find_server(cs, args.server) + else: + server = None + group = cs.ipgroups.create(args.name, server) + utils.print_dict(group._info) + + +@utils.arg('group', metavar='', help='Name or ID of group.') +def do_ipgroup_delete(cs, args): + """Delete an IP group.""" + _find_ipgroup(cs, args.group).delete() + + +@utils.arg('--fixed_ip', + dest='fixed_ip', + metavar='', + default=None, + help='Only match against fixed IP.') +@utils.arg('--reservation_id', + dest='reservation_id', + metavar='', + default=None, + help='Only return instances that match reservation_id.') +@utils.arg('--recurse_zones', + dest='recurse_zones', + metavar='<0|1>', + nargs='?', + type=int, + const=1, + default=0, + help='Recurse through all zones if set.') +@utils.arg('--ip', + dest='ip', + metavar='', + default=None, + help='Search with regular expression match by IP address') +@utils.arg('--ip6', + dest='ip6', + metavar='', + default=None, + help='Search with regular expression match by IPv6 address') +@utils.arg('--name', + dest='name', + metavar='', + default=None, + help='Search with regular expression match by name') +@utils.arg('--instance_name', + dest='instance_name', + metavar='', + default=None, + help='Search with regular expression match by instance name') +@utils.arg('--status', + dest='status', + metavar='', + default=None, + help='Search by server status') +@utils.arg('--flavor', + dest='flavor', + metavar='', + type=int, + default=None, + help='Search by flavor ID') +@utils.arg('--image', + dest='image', + type=int, + metavar='', + default=None, + help='Search by image ID') +@utils.arg('--host', + dest='host', + metavar='', + default=None, + help="Search by instances by hostname to which they are assigned") +def do_list(cs, args): + """List active servers.""" + recurse_zones = args.recurse_zones + search_opts = { + 'reservation_id': args.reservation_id, + 'fixed_ip': args.fixed_ip, + 'recurse_zones': recurse_zones, + 'ip': args.ip, + 'ip6': args.ip6, + 'name': args.name, + 'image': args.image, + 'flavor': args.flavor, + 'status': args.status, + 'host': args.host, + 'instance_name': args.instance_name} + if recurse_zones: + to_print = ['UUID', 'Name', 'Status', 'Public IP', 'Private IP'] + else: + to_print = ['ID', 'Name', 'Status', 'Public IP', 'Private IP'] + utils.print_list(cs.servers.list(search_opts=search_opts), + to_print) + + +@utils.arg('--hard', + dest='reboot_type', + action='store_const', + const=servers.REBOOT_HARD, + default=servers.REBOOT_SOFT, + help='Perform a hard reboot (instead of a soft one).') +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_reboot(cs, args): + """Reboot a server.""" + _find_server(cs, args.server).reboot(args.reboot_type) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('image', metavar='', help="Name or ID of new image.") +def do_rebuild(cs, args): + """Shutdown, re-image, and re-boot a server.""" + server = _find_server(cs, args.server) + image = _find_image(cs, args.image) + server.rebuild(image) + + +@utils.arg('server', metavar='', + help='Name (old name) or ID of server.') +@utils.arg('name', metavar='', help='New name for the server.') +def do_rename(cs, args): + """Rename a server.""" + _find_server(cs, args.server).update(name=args.name) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('flavor', metavar='', help="Name or ID of new flavor.") +def do_resize(cs, args): + """Resize a server.""" + server = _find_server(cs, args.server) + flavor = _find_flavor(cs, args.flavor) + server.resize(flavor) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('name', metavar='', help='Name of snapshot.') +@utils.arg('backup_type', metavar='', help='type of backup') +@utils.arg('rotation', type=int, metavar='', + help="Number of backups to retain. Used for backup image_type.") +def do_backup(cs, args): + """Backup a server.""" + server = _find_server(cs, args.server) + server.backup(args.name, args.backup_type, args.rotation) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_migrate(cs, args): + """Migrate a server.""" + _find_server(cs, args.server).migrate() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_pause(cs, args): + """Pause a server.""" + _find_server(cs, args.server).pause() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_unpause(cs, args): + """Unpause a server.""" + _find_server(cs, args.server).unpause() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_suspend(cs, args): + """Suspend a server.""" + _find_server(cs, args.server).suspend() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_resume(cs, args): + """Resume a server.""" + _find_server(cs, args.server).resume() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_rescue(cs, args): + """Rescue a server.""" + _find_server(cs, args.server).rescue() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_unrescue(cs, args): + """Unrescue a server.""" + _find_server(cs, args.server).unrescue() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_diagnostics(cs, args): + """Retrieve server diagnostics.""" + utils.print_dict(cs.servers.diagnostics(args.server)[1]) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_actions(cs, args): + """Retrieve server actions.""" + utils.print_list( + cs.servers.actions(args.server), + ["Created_At", "Action", "Error"]) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_resize_confirm(cs, args): + """Confirm a previous resize.""" + _find_server(cs, args.server).confirm_resize() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_resize_revert(cs, args): + """Revert a previous resize (and return to the previous VM).""" + _find_server(cs, args.server).revert_resize() + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_root_password(cs, args): + """ + Change the root password for a server. + """ + server = _find_server(cs, args.server) + p1 = getpass.getpass('New password: ') + p2 = getpass.getpass('Again: ') + if p1 != p2: + raise exceptions.CommandError("Passwords do not match.") + server.update(password=p1) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_show(cs, args): + """Show details about the given server.""" + s = _find_server(cs, args.server) + + info = s._info.copy() + addresses = info.pop('addresses') + for addrtype in addresses: + info['%s ip' % addrtype] = ', '.join(addresses[addrtype]) + + flavorId = info.get('flavorId', None) + if flavorId: + info['flavor'] = _find_flavor(cs, info.pop('flavorId')).name + imageId = info.get('imageId', None) + if imageId: + info['image'] = _find_image(cs, info.pop('imageId')).name + + utils.print_dict(info) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +def do_delete(cs, args): + """Immediately shut down and delete a server.""" + _find_server(cs, args.server).delete() + + +# --zone_username is required since --username is already used. +@utils.arg('zone', metavar='', help='ID of the zone', default=None) +@utils.arg('--api_url', dest='api_url', default=None, help='New URL.') +@utils.arg('--zone_username', dest='zone_username', default=None, + help='New zone username.') +@utils.arg('--zone_password', dest='zone_password', default=None, + help='New password.') +@utils.arg('--weight_offset', dest='weight_offset', default=None, + help='Child Zone weight offset.') +@utils.arg('--weight_scale', dest='weight_scale', default=None, + help='Child Zone weight scale.') +def do_zone(cs, args): + """Show or edit a child zone. No zone arg for this zone.""" + zone = cs.zones.get(args.zone) + + # If we have some flags, update the zone + zone_delta = {} + if args.api_url: + zone_delta['api_url'] = args.api_url + if args.zone_username: + zone_delta['username'] = args.zone_username + if args.zone_password: + zone_delta['password'] = args.zone_password + if args.weight_offset: + zone_delta['weight_offset'] = args.weight_offset + if args.weight_scale: + zone_delta['weight_scale'] = args.weight_scale + if zone_delta: + zone.update(**zone_delta) + else: + utils.print_dict(zone._info) + + +def do_zone_info(cs, args): + """Get this zones name and capabilities.""" + zone = cs.zones.info() + utils.print_dict(zone._info) + + +@utils.arg('zone_name', metavar='', + help='Name of the child zone being added.') +@utils.arg('api_url', metavar='', help="URL for the Zone's Auth API") +@utils.arg('--zone_username', metavar='', + help='Optional Authentication username. (Default=None)', + default=None) +@utils.arg('--zone_password', metavar='', + help='Authentication password. (Default=None)', + default=None) +@utils.arg('--weight_offset', metavar='', + help='Child Zone weight offset (Default=0.0))', + default=0.0) +@utils.arg('--weight_scale', metavar='', + help='Child Zone weight scale (Default=1.0).', + default=1.0) +def do_zone_add(cs, args): + """Add a new child zone.""" + zone = cs.zones.create(args.zone_name, args.api_url, + args.zone_username, args.zone_password, + args.weight_offset, args.weight_scale) + utils.print_dict(zone._info) + + +@utils.arg('zone', metavar='', help='Name or ID of the zone') +def do_zone_delete(cs, args): + """Delete a zone.""" + cs.zones.delete(args.zone) + + +def do_zone_list(cs, args): + """List the children of a zone.""" + utils.print_list(cs.zones.list(), ['ID', 'Name', 'Is Active', \ + 'API URL', 'Weight Offset', 'Weight Scale']) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('network_id', metavar='', help='Network ID.') +def do_add_fixed_ip(cs, args): + """Add new IP address to network.""" + server = _find_server(cs, args.server) + server.add_fixed_ip(args.network_id) + + +@utils.arg('server', metavar='', help='Name or ID of server.') +@utils.arg('address', metavar='
', help='IP Address.') +def do_remove_fixed_ip(cs, args): + """Remove an IP address from a server.""" + server = _find_server(cs, args.server) + server.remove_fixed_ip(args.address) + + +def _find_server(cs, server): + """Get a server by name or ID.""" + return utils.find_resource(cs.servers, server) + + +def _find_ipgroup(cs, group): + """Get an IP group by name or ID.""" + return utils.find_resource(cs.ipgroups, group) + + +def _find_image(cs, image): + """Get an image by name or ID.""" + return utils.find_resource(cs.images, image) + + +def _find_flavor(cs, flavor): + """Get a flavor by name, ID, or RAM size.""" + try: + return utils.find_resource(cs.flavors, flavor) + except exceptions.NotFound: + return cs.flavors.find(ram=flavor) diff --git a/v1_0/zones.py b/v1_0/zones.py new file mode 100644 index 00000000..6e75ed2a --- /dev/null +++ b/v1_0/zones.py @@ -0,0 +1,199 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Zone interface. +""" + +from novaclient import base +import base as local_base + + +class Weighting(base.Resource): + def __init__(self, manager, info, loaded=False): + self.name = "n/a" + super(Weighting, self).__init__(manager, info, loaded) + + def __repr__(self): + return "" % self.name + + def to_dict(self): + """Return the original info setting, which is a dict.""" + return self._info + + +class Zone(base.Resource): + def __init__(self, manager, info, loaded=False): + self.name = "n/a" + self.is_active = "n/a" + self.capabilities = "n/a" + super(Zone, self).__init__(manager, info, loaded) + + def __repr__(self): + return "" % self.api_url + + def delete(self): + """ + Delete a child zone. + """ + self.manager.delete(self) + + def update(self, api_url=None, username=None, password=None, + weight_offset=None, weight_scale=None): + """ + Update the name for this child zone. + + :param api_url: Update the child zone's API URL. + :param username: Update the child zone's username. + :param password: Update the child zone's password. + :param weight_offset: Update the child zone's weight offset. + :param weight_scale: Update the child zone's weight scale. + """ + self.manager.update(self, api_url, username, password, + weight_offset, weight_scale) + + +class ZoneManager(local_base.BootingManagerWithFind): + resource_class = Zone + + def info(self): + """ + Get info on this zone. + + :rtype: :class:`Zone` + """ + return self._get("/zones/info", "zone") + + def get(self, zone): + """ + Get a child zone. + + :param server: ID of the :class:`Zone` to get. + :rtype: :class:`Zone` + """ + return self._get("/zones/%s" % base.getid(zone), "zone") + + def list(self, detailed=True): + """ + Get a list of child zones. + :rtype: list of :class:`Zone` + """ + detail = "" + if detailed: + detail = "/detail" + return self._list("/zones%s" % detail, "zones") + + def create(self, zone_name, api_url, username, password, + weight_offset=0.0, weight_scale=1.0): + """ + Create a new child zone. + + :param zone_name: The child zone's name. + :param api_url: The child zone's auth URL. + :param username: The child zone's username. + :param password: The child zone's password. + :param weight_offset: The child zone's weight offset. + :param weight_scale: The child zone's weight scale. + """ + body = {"zone": { + "name": zone_name, + "api_url": api_url, + "username": username, + "password": password, + "weight_offset": weight_offset, + "weight_scale": weight_scale + }} + + return self._create("/zones", body, "zone") + + def boot(self, name, image, flavor, ipgroup=None, meta=None, files=None, + zone_blob=None, reservation_id=None, min_count=None, + max_count=None): + """ + Create (boot) a new server while being aware of Zones. + + :param name: Something to name the server. + :param image: The :class:`Image` to boot with. + :param flavor: The :class:`Flavor` to boot onto. + :param ipgroup: An initial :class:`IPGroup` for this server. + :param meta: A dict of arbitrary key/value metadata to store for this + server. A maximum of five entries is allowed, and both + keys and values must be 255 characters or less. + :param files: A dict of files to overrwrite on the server upon boot. + Keys are file names (i.e. ``/etc/passwd``) and values + are the file contents (either as a string or as a + file-like object). A maximum of five entries is allowed, + and each file must be 10k or less. + :param zone_blob: a single (encrypted) string which is used internally + by Nova for routing between Zones. Users cannot populate + this field. + :param reservation_id: a UUID for the set of servers being requested. + :param min_count: minimum number of servers to create. + :param max_count: maximum number of servers to create. + """ + if not min_count: + min_count = 1 + if not max_count: + max_count = min_count + return self._boot("/zones/boot", "reservation_id", name, image, flavor, + ipgroup=ipgroup, meta=meta, files=files, + zone_blob=zone_blob, reservation_id=reservation_id, + return_raw=True, min_count=min_count, + max_count=max_count) + + def select(self, *args, **kwargs): + """ + Given requirements for a new instance, select hosts + in this zone that best match those requirements. + """ + # 'specs' may be passed in as None, so change to an empty string. + specs = kwargs.get("specs") or "" + url = "/zones/select" + weighting_list = self._list(url, "weights", Weighting, body=specs) + return [wt.to_dict() for wt in weighting_list] + + def delete(self, zone): + """ + Delete a child zone. + """ + self._delete("/zones/%s" % base.getid(zone)) + + def update(self, zone, api_url=None, username=None, password=None, + weight_offset=None, weight_scale=None): + """ + Update the name or the api_url for a zone. + + :param zone: The :class:`Zone` (or its ID) to update. + :param api_url: Update the API URL. + :param username: Update the username. + :param password: Update the password. + :param weight_offset: Update the child zone's weight offset. + :param weight_scale: Update the child zone's weight scale. + """ + + body = {"zone": {}} + if api_url: + body["zone"]["api_url"] = api_url + if username: + body["zone"]["username"] = username + if password: + body["zone"]["password"] = password + if weight_offset: + body["zone"]["weight_offset"] = weight_offset + if weight_scale: + body["zone"]["weight_scale"] = weight_scale + if not len(body["zone"]): + return + self._update("/zones/%s" % base.getid(zone), body) diff --git a/vmdatabase.py b/vmdatabase.py index 75d8d71f..266b74a1 100644 --- a/vmdatabase.py +++ b/vmdatabase.py @@ -1,3 +1,23 @@ +#!/usr/bin/env python + +# Keep track of VMs used by the devstack gate test. + +# Copyright (C) 2011-2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + import sqlite3 import os import time @@ -5,102 +25,294 @@ import time # States: # The cloud provider is building this machine. We have an ID, but it's # not ready for use. -BUILDING=1 +BUILDING = 1 # The machine is ready for use. -READY=2 +READY = 2 # This can mean in-use, or used but complete. We don't actually need to -# distinguish between those states -- we'll just delete a machine 24 hours +# distinguish between those states -- we'll just delete a machine 24 hours # after it transitions into the USED state. -USED=3 +USED = 3 # An error state, should just try to delete it. -ERROR=4 +ERROR = 4 +# Keep this machine indefinitely +HOLD = 5 + +from sqlalchemy import Table, Column, Boolean, Integer, String, MetaData, ForeignKey, UniqueConstraint, Index, create_engine, and_, or_ +from sqlalchemy.orm import mapper, relation +from sqlalchemy.orm.session import Session, sessionmaker + +metadata = MetaData() +provider_table = Table('provider', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(255), index=True, unique=True), + Column('max_servers', Integer), # Max total number of servers for this provider + Column('giftable', Boolean), # May we give failed vms from this provider to developers? + Column('nova_api_version', String(8)), # 1.0 or 1.1 + Column('nova_rax_auth', Boolean), # novaclient doesn't discover this itself + Column('nova_username', String(255)), + Column('nova_api_key', String(255)), + Column('nova_auth_url', String(255)), # Authentication URL + Column('nova_project_id', String(255)), # Project id to use at authn + Column('nova_service_type', String(255)), # endpoint selection: service type (Null for default) + Column('nova_service_region', String(255)), # endpoint selection: service region (Null for default) + Column('nova_service_name', String(255)), # endpoint selection: Endpoint name (Null for default) + ) +base_image_table = Table('base_image', metadata, + Column('id', Integer, primary_key=True), + Column('provider_id', Integer, ForeignKey('provider.id'), index=True, nullable=False), + Column('name', String(255)), # Image name (oneiric, precise, etc). + Column('external_id', String(255)), # Provider assigned id for this image + Column('min_ready', Integer), # Min number of servers to keep ready for this provider/image + Column('min_ram', Integer), # amount of ram to select for servers with this image + #active? + ) +snapshot_image_table = Table('snapshot_image', metadata, + Column('id', Integer, primary_key=True), + Column('name', String(255)), + Column('base_image_id', Integer, ForeignKey('base_image.id'), index=True, nullable=False), + Column('version', Integer), # Version indicator (timestamp) + Column('external_id', String(255)), # Provider assigned id for this image + Column('server_external_id', String(255)), # Provider assigned id of the server used to create the snapshot + Column('state', Integer), # One of the above values + Column('state_time', Integer), # Time of last state change + ) +machine_table = Table('machine', metadata, + Column('id', Integer, primary_key=True), + Column('base_image_id', Integer, ForeignKey('base_image.id'), index=True, nullable=False), + Column('external_id', String(255)), # Provider assigned id for this machine + Column('name', String(255)), # Machine name + Column('ip', String(255)), # Primary IP address + Column('user', String(255)), # Username if ssh keys have been installed, or NULL + Column('state', Integer), # One of the above values + Column('state_time', Integer), # Time of last state change + ) + + +class Provider(object): + def __init__(self, name, driver, username, api_key, giftable): + self.name = name + self.driver = driver + self.username = username + self.api_key = api_key + self.giftable = giftable + + def delete(self): + session = Session.object_session(self) + session.delete(self) + session.commit() + + def newBaseImage(self, *args, **kwargs): + new = BaseImage(*args, **kwargs) + new.provider = self + session = Session.object_session(self) + session.commit() + return new + + def getBaseImage(self, name): + session = Session.object_session(self) + return session.query(BaseImage).filter(and_( + base_image_table.c.name == name, + base_image_table.c.provider_id == self.id)).first() + + def _machines(self): + session = Session.object_session(self) + return session.query(Machine).filter(and_( + machine_table.c.base_image_id == base_image_table.c.id, + base_image_table.c.provider_id == self.id)).order_by( + machine_table.c.state_time) + + @property + def machines(self): + return self._machines().all() + + @property + def building_machines(self): + return self._machines().filter(machine_table.c.state == BUILDING).all() + + @property + def ready_machines(self): + return self._machines().filter(machine_table.c.state == READY).all() + + +class BaseImage(object): + def __init__(self, name, external_id): + self.name = name + self.external_id = external_id + + def delete(self): + session = Session.object_session(self) + session.delete(self) + session.commit() + + def newSnapshotImage(self, *args, **kwargs): + new = SnapshotImage(*args, **kwargs) + new.base_image = self + session = Session.object_session(self) + session.commit() + return new + + def newMachine(self, *args, **kwargs): + new = Machine(*args, **kwargs) + new.base_image = self + session = Session.object_session(self) + session.commit() + return new + + @property + def ready_snapshot_images(self): + session = Session.object_session(self) + return session.query(SnapshotImage).filter(and_( + snapshot_image_table.c.base_image_id == self.id, + snapshot_image_table.c.state == READY)).order_by( + snapshot_image_table.c.version).all() + + @property + def current_snapshot(self): + if not self.ready_snapshot_images: + return None + return self.ready_snapshot_images[-1] + + def _machines(self): + session = Session.object_session(self) + return session.query(Machine).filter( + machine_table.c.base_image_id == self.id).order_by( + machine_table.c.state_time) + + @property + def building_machines(self): + return self._machines().filter(machine_table.c.state == BUILDING).all() + + @property + def ready_machines(self): + return self._machines().filter(machine_table.c.state == READY).all() + + +class SnapshotImage(object): + def __init__(self, name, version, external_id, server_external_id, state=BUILDING): + self.name = name + self.version = version + self.external_id = external_id + self.server_external_id = server_external_id + self.state = state + + def delete(self): + session = Session.object_session(self) + session.delete(self) + session.commit() + + @property + def state(self): + return self._state + + @state.setter + def state(self, state): + self._state = state + self.state_time = int(time.time()) + session = Session.object_session(self) + if session: + session.commit() + + +class Machine(object): + def __init__(self, name, external_id, ip=None, user=None, state=BUILDING): + self.name = name + self.external_id = external_id + self.ip = ip + self.user = user + self.state = state + + def delete(self): + session = Session.object_session(self) + session.delete(self) + session.commit() + + @property + def state(self): + return self._state + + @state.setter + def state(self, state): + self._state = state + self.state_time = int(time.time()) + session = Session.object_session(self) + if session: + session.commit() + + +mapper(Machine, machine_table, properties=dict( + _state=machine_table.c.state, + )) + +mapper(SnapshotImage, snapshot_image_table, properties=dict( + _state=snapshot_image_table.c.state, + )) + +mapper(BaseImage, base_image_table, properties=dict( + snapshot_images=relation(SnapshotImage, + order_by=snapshot_image_table.c.version, + cascade='all, delete-orphan', + backref='base_image'), + machines=relation(Machine, + order_by=machine_table.c.state_time, + cascade='all, delete-orphan', + backref='base_image'))) + +mapper(Provider, provider_table, properties=dict( + base_images=relation(BaseImage, + order_by=base_image_table.c.name, + cascade='all, delete-orphan', + backref='provider'))) -# Columns: -# state: one of the above values -# state_time: the time of transition into that state -# user: set if the machine is given to a user -# id: identifier from cloud provider -# name: machine name -# ip: machine ip -# uuid: uuid from libcloud -# provider: libcloud driver for this server -# image: name of image this server is based on class VMDatabase(object): def __init__(self, path=os.path.expanduser("~/vm.db")): - # Set isolation_level = None, which means "autocommit" mode - # but more importantly lets you manage transactions manually - # without the isolation emulation getting in your way. - # Most of our writes can be autocomitted, and the one(s) - # that can't, we'll set up the transaction around the critical - # section. - if not os.path.exists(path): - conn = sqlite3.connect(path, isolation_level=None) - conn.execute("""create table machines - (provider text, id int, image text, - name text, ip text, uuid text, - state_time int, state int, user text)""") - del conn - self.conn = sqlite3.connect(path, timeout=30, isolation_level=None) - # This turns the returned rows into objects that are like lists - # and dicts at the same time: - self.conn.row_factory = sqlite3.Row + engine = create_engine('sqlite:///%s' % path, echo=False) + metadata.create_all(engine) + Session = sessionmaker(bind=engine, autoflush=True, autocommit=False) + self.session = Session() - def addMachine(self, provider, mid, image, name, ip, uuid): - self.conn.execute("""insert into machines - (provider, id, image, name, ip, - uuid, state_time, state) - values (?, ?, ?, ?, ?, ?, ?, ?)""", - (provider, mid, image, name, ip, uuid, - int(time.time()), BUILDING)) - - def delMachine(self, uuid): - self.conn.execute("delete from machines where uuid=?", (uuid,)) + def print_state(self): + for provider in self.getProviders(): + print 'Provider:', provider.name + for base_image in provider.base_images: + print ' Base image:', base_image.name + for snapshot_image in base_image.snapshot_images: + print ' Snapshot:', snapshot_image.name, snapshot_image.state + for machine in base_image.machines: + print ' Machine:', machine.id, machine.name, machine.state, machine.state_time, machine.ip - def setMachineUser(self, uuid, user): - self.conn.execute("update machines set user=? where uuid=?", - (user, uuid)) + def abort(self): + self.session.rollback() - def setMachineState(self, uuid, state): - self.conn.execute("""update machines set state=?, state_time=? - where uuid=?""", - (state, int(time.time()), uuid)) + def commit(self): + self.session.commit() - def getMachines(self): - return self.conn.execute("select * from machines order by state_time") + def delete(self, obj): + self.session.delete(obj) - def getMachine(self, uuid): - for x in self.conn.execute("select * from machines where uuid=?", - (uuid,)): - return x + def getProviders(self): + return self.session.query(Provider).all() - def getMachineForUse(self): + def getProvider(self, name): + return self.session.query(Provider).filter_by(name=name)[0] + + def getMachine(self, id): + return self.session.query(Machine).filter_by(id=id)[0] + + def getMachineForUse(self, image_name): """Atomically find a machine that is ready for use, and update its state.""" - self.conn.execute("begin exclusive transaction") - ret = None - for m in self.getMachines(): - if m['state']==READY: - self.setMachineState(m['uuid'], USED) - ret = m - break - self.conn.execute("commit") - return ret + image = None + for machine in self.session.query(Machine).filter( + machine_table.c.state == READY).order_by( + machine_table.c.state_time): + if machine.base_image.name == image_name: + machine.state = USED + self.commit() + return machine + raise Exception("No machine found for image %s" % image_name) -if __name__=='__main__': - db = VMDatabase("/tmp/vm.db") - db.addMachine('rackspace', 1, 'devstack', 'foo', '1.2.3.4', 'uuid1') - db.setMachineState('uuid1', READY) - db.addMachine('rackspace', 2, 'devstack', 'foo2', '1.2.3.4', 'uuid2') - db.setMachineState('uuid2', READY) - m = db.getMachineForUse() - print 'got machine' - print m - db.setMachineUser(m['uuid'], 'jeblair') - print db.getMachines() - print db.getMachine(1) - print 'waiting to delete' - time.sleep(2) - db.delMachine('uuid1') - db.delMachine('uuid2') +if __name__ == '__main__': + db = VMDatabase() + db.print_state()