diff --git a/bin/nova-manage b/bin/nova-manage index 7dfc3c045..cb4d18614 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -55,6 +55,8 @@ import datetime import gettext +import glob +import json import os import re import sys @@ -81,7 +83,7 @@ from nova import log as logging from nova import quota from nova import rpc from nova import utils -from nova.api.ec2.cloud import ec2_id_to_id +from nova.api.ec2 import ec2utils from nova.auth import manager from nova.cloudpipe import pipelib from nova.compute import instance_types @@ -94,6 +96,7 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') +flags.DECLARE('images_path', 'nova.image.local') flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) @@ -104,7 +107,7 @@ def param2id(object_id): args: [object_id], e.g. 'vol-0000000a' or 'volume-0000000a' or '10' """ if '-' in object_id: - return ec2_id_to_id(object_id) + return ec2utils.ec2_id_to_id(object_id) else: return int(object_id) @@ -273,7 +276,7 @@ def _db_error(caught_exception): print caught_exception print _("The above error may show that the database has not " "been created.\nPlease create a database using " - "nova-manage sync db before running this command.") + "'nova-manage db sync' before running this command.") exit(1) @@ -434,6 +437,8 @@ class ProjectCommands(object): "been created.\nPlease create a database by running a " "nova-api server on this host.") +AccountCommands = ProjectCommands + class FixedIpCommands(object): """Class for managing fixed ip.""" @@ -744,8 +749,158 @@ class InstanceTypeCommands(object): self._print_instance_types(name, inst_types) +class ImageCommands(object): + """Methods for dealing with a cloud in an odd state""" + + def __init__(self, *args, **kwargs): + self.image_service = utils.import_object(FLAGS.image_service) + + def _register(self, image_type, disk_format, container_format, + path, owner, name=None, is_public='T', + architecture='x86_64', kernel_id=None, ramdisk_id=None): + meta = {'is_public': True, + 'name': name, + 'disk_format': disk_format, + 'container_format': container_format, + 'properties': {'image_state': 'available', + 'owner': owner, + 'type': image_type, + 'architecture': architecture, + 'image_location': 'local', + 'is_public': (is_public == 'T')}} + print image_type, meta + if kernel_id: + meta['properties']['kernel_id'] = int(kernel_id) + if ramdisk_id: + meta['properties']['ramdisk_id'] = int(ramdisk_id) + elevated = context.get_admin_context() + try: + with open(path) as ifile: + image = self.image_service.create(elevated, meta, ifile) + new = image['id'] + print _("Image registered to %(new)s (%(new)08x).") % locals() + return new + except Exception as exc: + print _("Failed to register %(path)s: %(exc)s") % locals() + + def all_register(self, image, kernel, ramdisk, owner, name=None, + is_public='T', architecture='x86_64'): + """Uploads an image, kernel, and ramdisk into the image_service + arguments: image kernel ramdisk owner [name] [is_public='T'] + [architecture='x86_64']""" + kernel_id = self.kernel_register(kernel, owner, None, + is_public, architecture) + ramdisk_id = self.ramdisk_register(ramdisk, owner, None, + is_public, architecture) + self.image_register(image, owner, name, is_public, + architecture, kernel_id, ramdisk_id) + + def image_register(self, path, owner, name=None, is_public='T', + architecture='x86_64', kernel_id=None, ramdisk_id=None, + disk_format='ami', container_format='ami'): + """Uploads an image into the image_service + arguments: path owner [name] [is_public='T'] [architecture='x86_64'] + [kernel_id=None] [ramdisk_id=None] + [disk_format='ami'] [container_format='ami']""" + return self._register('machine', disk_format, container_format, path, + owner, name, is_public, architecture, + kernel_id, ramdisk_id) + + def kernel_register(self, path, owner, name=None, is_public='T', + architecture='x86_64'): + """Uploads a kernel into the image_service + arguments: path owner [name] [is_public='T'] [architecture='x86_64'] + """ + return self._register('kernel', 'aki', 'aki', path, owner, name, + is_public, architecture) + + def ramdisk_register(self, path, owner, name=None, is_public='T', + architecture='x86_64'): + """Uploads a ramdisk into the image_service + arguments: path owner [name] [is_public='T'] [architecture='x86_64'] + """ + return self._register('ramdisk', 'ari', 'ari', path, owner, name, + is_public, architecture) + + def _lookup(self, old_image_id): + try: + internal_id = ec2utils.ec2_id_to_id(old_image_id) + image = self.image_service.show(context, internal_id) + except exception.NotFound: + image = self.image_service.show_by_name(context, old_image_id) + return image['id'] + + def _old_to_new(self, old): + mapping = {'machine': 'ami', + 'kernel': 'aki', + 'ramdisk': 'ari'} + container_format = mapping[old['type']] + disk_format = container_format + new = {'disk_format': disk_format, + 'container_format': container_format, + 'is_public': True, + 'name': old['imageId'], + 'properties': {'image_state': old['imageState'], + 'owner': old['imageOwnerId'], + 'architecture': old['architecture'], + 'type': old['type'], + 'image_location': old['imageLocation'], + 'is_public': old['isPublic']}} + if old.get('kernelId'): + new['properties']['kernel_id'] = self._lookup(old['kernelId']) + if old.get('ramdiskId'): + new['properties']['ramdisk_id'] = self._lookup(old['ramdiskId']) + return new + + def _convert_images(self, images): + elevated = context.get_admin_context() + for image_path, image_metadata in images.iteritems(): + meta = self._old_to_new(image_metadata) + old = meta['name'] + try: + with open(image_path) as ifile: + image = self.image_service.create(elevated, meta, ifile) + new = image['id'] + print _("Image %(old)s converted to " \ + "%(new)s (%(new)08x).") % locals() + except Exception as exc: + print _("Failed to convert %(old)s: %(exc)s") % locals() + + def convert(self, directory): + """Uploads old objectstore images in directory to new service + arguments: directory""" + machine_images = {} + other_images = {} + directory = os.path.abspath(directory) + # NOTE(vish): If we're importing from the images path dir, attempt + # to move the files out of the way before importing + # so we aren't writing to the same directory. This + # may fail if the dir was a mointpoint. + if (FLAGS.image_service == 'nova.image.local.LocalImageService' + and directory == os.path.abspath(FLAGS.images_path)): + new_dir = "%s_bak" % directory + os.move(directory, new_dir) + os.mkdir(directory) + directory = new_dir + for fn in glob.glob("%s/*/info.json" % directory): + try: + image_path = os.path.join(fn.rpartition('/')[0], 'image') + with open(fn) as metadata_file: + image_metadata = json.load(metadata_file) + if image_metadata['type'] == 'machine': + machine_images[image_path] = image_metadata + else: + other_images[image_path] = image_metadata + except Exception as exc: + print _("Failed to load %(fn)s.") % locals() + # NOTE(vish): do kernels and ramdisks first so images + self._convert_images(other_images) + self._convert_images(machine_images) + + CATEGORIES = [ ('user', UserCommands), + ('account', AccountCommands), ('project', ProjectCommands), ('role', RoleCommands), ('shell', ShellCommands), @@ -758,6 +913,7 @@ CATEGORIES = [ ('db', DbCommands), ('volume', VolumeCommands), ('instance_type', InstanceTypeCommands), + ('image', ImageCommands), ('flavor', InstanceTypeCommands)] diff --git a/doc/source/man/novamanage.rst b/doc/source/man/novamanage.rst index 17ba91bef..1d8446f08 100644 --- a/doc/source/man/novamanage.rst +++ b/doc/source/man/novamanage.rst @@ -173,7 +173,10 @@ Nova Floating IPs ``nova-manage floating create `` Creates floating IP addresses for the named host by the given range. - floating delete Deletes floating IP addresses in the range given. + +``nova-manage floating delete `` + + Deletes floating IP addresses in the range given. ``nova-manage floating list`` @@ -193,7 +196,7 @@ Nova Flavor ``nova-manage flavor create <(optional) swap> <(optional) RXTX Quota> <(optional) RXTX Cap>`` creates a flavor with the following positional arguments: - * memory (expressed in megabytes) + * memory (expressed in megabytes) * vcpu(s) (integer) * local storage (expressed in gigabytes) * flavorid (unique integer) @@ -209,12 +212,33 @@ Nova Flavor Purges the flavor with the name . This removes this flavor from the database. - Nova Instance_type ~~~~~~~~~~~~~~~~~~ The instance_type command is provided as an alias for the flavor command. All the same subcommands and arguments from nova-manage flavor can be used. +Nova Images +~~~~~~~~~~~ + +``nova-manage image image_register `` + + Registers an image with the image service. + +``nova-manage image kernel_register `` + + Registers a kernel with the image service. + +``nova-manage image ramdisk_register `` + + Registers a ramdisk with the image service. + +``nova-manage image all_register `` + + Registers an image kernel and ramdisk with the image service. + +``nova-manage image convert `` + + Converts all images in directory from the old (Bexar) format to the new format. FILES ======== diff --git a/doc/source/runnova/nova.manage.rst b/doc/source/runnova/nova.manage.rst index 0e9a29b6b..0636e5752 100644 --- a/doc/source/runnova/nova.manage.rst +++ b/doc/source/runnova/nova.manage.rst @@ -182,6 +182,29 @@ Nova Floating IPs Displays a list of all floating IP addresses. +Nova Images +~~~~~~~~~~~ + +``nova-manage image image_register `` + + Registers an image with the image service. + +``nova-manage image kernel_register `` + + Registers a kernel with the image service. + +``nova-manage image ramdisk_register `` + + Registers a ramdisk with the image service. + +``nova-manage image all_register `` + + Registers an image kernel and ramdisk with the image service. + +``nova-manage image convert `` + + Converts all images in directory from the old (Bexar) format to the new format. + Concept: Flags -------------- diff --git a/nova/flags.py b/nova/flags.py index 213d4d4e1..9123e9ac7 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -348,7 +348,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.s3.S3ImageService', +DEFINE_string('image_service', 'nova.image.local.LocalImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), @@ -356,3 +356,7 @@ DEFINE_string('host', socket.gethostname(), DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') + +DEFINE_string('zone_name', 'nova', 'name of this zone') +DEFINE_string('zone_capabilities', 'kypervisor:xenserver;os:linux', + 'Key/Value tags which represent capabilities of this zone') diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py new file mode 100644 index 000000000..2405f1343 --- /dev/null +++ b/nova/scheduler/api.py @@ -0,0 +1,49 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Handles all requests relating to schedulers. +""" + +from nova import flags +from nova import log as logging +from nova import rpc + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.scheduler.api') + + +class API(object): + """API for interacting with the scheduler.""" + + def _call_scheduler(self, method, context, params=None): + """Generic handler for RPC calls to the scheduler. + + :param params: Optional dictionary of arguments to be passed to the + scheduler worker + + :retval: Result returned by scheduler worker + """ + if not params: + params = {} + queue = FLAGS.scheduler_topic + kwargs = {'method': method, 'args': params} + return rpc.call(context, queue, kwargs) + + def get_zone_list(self, context): + items = self._call_scheduler('get_zone_list', context) + for item in items: + item['api_url'] = item['api_url'].replace('\\/', '/') + return items diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index e9b47512e..c94397210 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -29,6 +29,7 @@ from nova import log as logging from nova import manager from nova import rpc from nova import utils +from nova.scheduler import zone_manager LOG = logging.getLogger('nova.scheduler.manager') FLAGS = flags.FLAGS @@ -43,12 +44,21 @@ class SchedulerManager(manager.Manager): if not scheduler_driver: scheduler_driver = FLAGS.scheduler_driver self.driver = utils.import_object(scheduler_driver) + self.zone_manager = zone_manager.ZoneManager() super(SchedulerManager, self).__init__(*args, **kwargs) def __getattr__(self, key): """Converts all method calls to use the schedule method""" return functools.partial(self._schedule, key) + def periodic_tasks(self, context=None): + """Poll child zones periodically to get status.""" + self.zone_manager.ping(context) + + def get_zone_list(self, context=None): + """Get a list of zones from the ZoneManager.""" + return self.zone_manager.get_zone_list() + def _schedule(self, method, context, topic, *args, **kwargs): """Tries to call schedule_* method on the driver to retrieve host. diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py new file mode 100644 index 000000000..edf9000cc --- /dev/null +++ b/nova/scheduler/zone_manager.py @@ -0,0 +1,143 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +ZoneManager oversees all communications with child Zones. +""" + +import novaclient +import thread +import traceback + +from datetime import datetime +from eventlet import greenpool + +from nova import db +from nova import flags +from nova import log as logging + +FLAGS = flags.FLAGS +flags.DEFINE_integer('zone_db_check_interval', 60, + 'Seconds between getting fresh zone info from db.') +flags.DEFINE_integer('zone_failures_to_offline', 3, + 'Number of consecutive errors before marking zone offline') + + +class ZoneState(object): + """Holds the state of all connected child zones.""" + def __init__(self): + self.is_active = True + self.name = None + self.capabilities = None + self.attempt = 0 + self.last_seen = datetime.min + self.last_exception = None + self.last_exception_time = None + + def update_credentials(self, zone): + """Update zone credentials from db""" + self.zone_id = zone.id + self.api_url = zone.api_url + self.username = zone.username + self.password = zone.password + + def update_metadata(self, zone_metadata): + """Update zone metadata after successful communications with + child zone.""" + self.last_seen = datetime.now() + self.attempt = 0 + self.name = zone_metadata["name"] + self.capabilities = zone_metadata["capabilities"] + self.is_active = True + + def to_dict(self): + return dict(name=self.name, capabilities=self.capabilities, + is_active=self.is_active, api_url=self.api_url, + id=self.zone_id) + + def log_error(self, exception): + """Something went wrong. Check to see if zone should be + marked as offline.""" + self.last_exception = exception + self.last_exception_time = datetime.now() + api_url = self.api_url + logging.warning(_("'%(exception)s' error talking to " + "zone %(api_url)s") % locals()) + + max_errors = FLAGS.zone_failures_to_offline + self.attempt += 1 + if self.attempt >= max_errors: + self.is_active = False + logging.error(_("No answer from zone %(api_url)s " + "after %(max_errors)d " + "attempts. Marking inactive.") % locals()) + + +def _call_novaclient(zone): + """Call novaclient. Broken out for testing purposes.""" + client = novaclient.OpenStack(zone.username, zone.password, zone.api_url) + return client.zones.info()._info + + +def _poll_zone(zone): + """Eventlet worker to poll a zone.""" + logging.debug(_("Polling zone: %s") % zone.api_url) + try: + zone.update_metadata(_call_novaclient(zone)) + except Exception, e: + zone.log_error(traceback.format_exc()) + + +class ZoneManager(object): + """Keeps the zone states updated.""" + def __init__(self): + self.last_zone_db_check = datetime.min + self.zone_states = {} + self.green_pool = greenpool.GreenPool() + + def get_zone_list(self): + """Return the list of zones we know about.""" + return [zone.to_dict() for zone in self.zone_states.values()] + + def _refresh_from_db(self, context): + """Make our zone state map match the db.""" + # Add/update existing zones ... + zones = db.zone_get_all(context) + existing = self.zone_states.keys() + db_keys = [] + for zone in zones: + db_keys.append(zone.id) + if zone.id not in existing: + self.zone_states[zone.id] = ZoneState() + self.zone_states[zone.id].update_credentials(zone) + + # Cleanup zones removed from db ... + keys = self.zone_states.keys() # since we're deleting + for zone_id in keys: + if zone_id not in db_keys: + del self.zone_states[zone_id] + + def _poll_zones(self, context): + """Try to connect to each child zone and get update.""" + self.green_pool.imap(_poll_zone, self.zone_states.values()) + + def ping(self, context=None): + """Ping should be called periodically to update zone status.""" + diff = datetime.now() - self.last_zone_db_check + if diff.seconds >= FLAGS.zone_db_check_interval: + logging.debug(_("Updating zone cache from db.")) + self.last_zone_db_check = datetime.now() + self._refresh_from_db(context) + self._poll_zones(context) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index cbd949477..5d7ca98b5 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -32,6 +32,7 @@ flags.DECLARE('fake_network', 'nova.network.manager') FLAGS.network_size = 8 FLAGS.num_networks = 2 FLAGS.fake_network = True +FLAGS.image_service = 'nova.image.local.LocalImageService' flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') flags.DECLARE('iscsi_num_targets', 'nova.volume.driver') diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index b195fa520..cf8ee7eff 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -38,6 +38,8 @@ from nova import test from nova.auth import manager from nova.compute import power_state from nova.api.ec2 import cloud +from nova.api.ec2 import ec2utils +from nova.image import local from nova.objectstore import image @@ -76,6 +78,12 @@ class CloudTestCase(test.TestCase): project=self.project) host = self.network.get_network_host(self.context.elevated()) + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} + + self.stubs.Set(local.LocalImageService, 'show', fake_show) + self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) + def tearDown(self): network_ref = db.project_get_network(self.context, self.project.id) @@ -122,7 +130,7 @@ class CloudTestCase(test.TestCase): self.cloud.allocate_address(self.context) inst = db.instance_create(self.context, {'host': self.compute.host}) fixed = self.network.allocate_fixed_ip(self.context, inst['id']) - ec2_id = cloud.id_to_ec2_id(inst['id']) + ec2_id = ec2utils.id_to_ec2_id(inst['id']) self.cloud.associate_address(self.context, instance_id=ec2_id, public_ip=address) @@ -158,12 +166,12 @@ class CloudTestCase(test.TestCase): vol2 = db.volume_create(self.context, {}) result = self.cloud.describe_volumes(self.context) self.assertEqual(len(result['volumeSet']), 2) - volume_id = cloud.id_to_ec2_id(vol2['id'], 'vol-%08x') + volume_id = ec2utils.id_to_ec2_id(vol2['id'], 'vol-%08x') result = self.cloud.describe_volumes(self.context, volume_id=[volume_id]) self.assertEqual(len(result['volumeSet']), 1) self.assertEqual( - cloud.ec2_id_to_id(result['volumeSet'][0]['volumeId']), + ec2utils.ec2_id_to_id(result['volumeSet'][0]['volumeId']), vol2['id']) db.volume_destroy(self.context, vol1['id']) db.volume_destroy(self.context, vol2['id']) @@ -188,8 +196,10 @@ class CloudTestCase(test.TestCase): def test_describe_instances(self): """Makes sure describe_instances works and filters results.""" inst1 = db.instance_create(self.context, {'reservation_id': 'a', + 'image_id': 1, 'host': 'host1'}) inst2 = db.instance_create(self.context, {'reservation_id': 'a', + 'image_id': 1, 'host': 'host2'}) comp1 = db.service_create(self.context, {'host': 'host1', 'availability_zone': 'zone1', @@ -200,7 +210,7 @@ class CloudTestCase(test.TestCase): result = self.cloud.describe_instances(self.context) result = result['reservationSet'][0] self.assertEqual(len(result['instancesSet']), 2) - instance_id = cloud.id_to_ec2_id(inst2['id']) + instance_id = ec2utils.id_to_ec2_id(inst2['id']) result = self.cloud.describe_instances(self.context, instance_id=[instance_id]) result = result['reservationSet'][0] @@ -215,10 +225,9 @@ class CloudTestCase(test.TestCase): db.service_destroy(self.context, comp2['id']) def test_console_output(self): - image_id = FLAGS.default_image instance_type = FLAGS.default_instance_type max_count = 1 - kwargs = {'image_id': image_id, + kwargs = {'image_id': 'ami-1', 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) @@ -234,8 +243,7 @@ class CloudTestCase(test.TestCase): greenthread.sleep(0.3) def test_ajax_console(self): - image_id = FLAGS.default_image - kwargs = {'image_id': image_id} + kwargs = {'image_id': 'ami-1'} rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] greenthread.sleep(0.3) @@ -347,7 +355,7 @@ class CloudTestCase(test.TestCase): def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) - ec2_id = cloud.id_to_ec2_id(inst['id']) + ec2_id = ec2utils.id_to_ec2_id(inst['id']) self.cloud.update_instance(self.context, ec2_id, display_name='c00l 1m4g3') inst = db.instance_get(self.context, inst['id']) @@ -365,7 +373,7 @@ class CloudTestCase(test.TestCase): def test_update_of_volume_display_fields(self): vol = db.volume_create(self.context, {}) self.cloud.update_volume(self.context, - cloud.id_to_ec2_id(vol['id'], 'vol-%08x'), + ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x'), display_name='c00l v0lum3') vol = db.volume_get(self.context, vol['id']) self.assertEqual('c00l v0lum3', vol['display_name']) @@ -374,7 +382,7 @@ class CloudTestCase(test.TestCase): def test_update_of_volume_wont_update_private_fields(self): vol = db.volume_create(self.context, {}) self.cloud.update_volume(self.context, - cloud.id_to_ec2_id(vol['id'], 'vol-%08x'), + ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x'), mountpoint='/not/here') vol = db.volume_get(self.context, vol['id']) self.assertEqual(None, vol['mountpoint']) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 58493d7ac..643b2e93a 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -31,7 +31,7 @@ from nova import test from nova import utils from nova.auth import manager from nova.compute import instance_types - +from nova.image import local LOG = logging.getLogger('nova.tests.compute') FLAGS = flags.FLAGS @@ -52,6 +52,11 @@ class ComputeTestCase(test.TestCase): self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.RequestContext('fake', 'fake', False) + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} + + self.stubs.Set(local.LocalImageService, 'show', fake_show) + def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) @@ -60,7 +65,7 @@ class ComputeTestCase(test.TestCase): def _create_instance(self, params={}): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 49ff24413..d47c70d88 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -57,7 +57,7 @@ class ConsoleTestCase(test.TestCase): inst = {} #inst['host'] = self.host #inst['name'] = 'instance-1234' - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 85bfcfd85..80e4d2e1f 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -93,8 +93,7 @@ class DirectTestCase(test.TestCase): class DirectCloudTestCase(test_cloud.CloudTestCase): def setUp(self): super(DirectCloudTestCase, self).setUp() - compute_handle = compute.API(image_service=self.cloud.image_service, - network_api=self.cloud.network_api, + compute_handle = compute.API(network_api=self.cloud.network_api, volume_api=self.cloud.volume_api) direct.register_service('compute', compute_handle) self.router = direct.JsonParamsMiddleware(direct.Router()) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 19099ff4c..53e35ce7e 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -29,11 +29,153 @@ from nova import log as logging from nova import test from nova import utils from nova.auth import manager +from nova.network import linux_net FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.network') +class IptablesManagerTestCase(test.TestCase): + sample_filter = ['#Generated by iptables-save on Fri Feb 18 15:17:05 2011', + '*filter', + ':INPUT ACCEPT [2223527:305688874]', + ':FORWARD ACCEPT [0:0]', + ':OUTPUT ACCEPT [2172501:140856656]', + ':nova-compute-FORWARD - [0:0]', + ':nova-compute-INPUT - [0:0]', + ':nova-compute-local - [0:0]', + ':nova-compute-OUTPUT - [0:0]', + ':nova-filter-top - [0:0]', + '-A FORWARD -j nova-filter-top ', + '-A OUTPUT -j nova-filter-top ', + '-A nova-filter-top -j nova-compute-local ', + '-A INPUT -j nova-compute-INPUT ', + '-A OUTPUT -j nova-compute-OUTPUT ', + '-A FORWARD -j nova-compute-FORWARD ', + '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ', + '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ', + '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ', + '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ', + '-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT ', + '-A FORWARD -i virbr0 -o virbr0 -j ACCEPT ', + '-A FORWARD -o virbr0 -j REJECT --reject-with ' + 'icmp-port-unreachable ', + '-A FORWARD -i virbr0 -j REJECT --reject-with ' + 'icmp-port-unreachable ', + 'COMMIT', + '# Completed on Fri Feb 18 15:17:05 2011'] + + sample_nat = ['# Generated by iptables-save on Fri Feb 18 15:17:05 2011', + '*nat', + ':PREROUTING ACCEPT [3936:762355]', + ':INPUT ACCEPT [2447:225266]', + ':OUTPUT ACCEPT [63491:4191863]', + ':POSTROUTING ACCEPT [63112:4108641]', + ':nova-compute-OUTPUT - [0:0]', + ':nova-compute-floating-ip-snat - [0:0]', + ':nova-compute-SNATTING - [0:0]', + ':nova-compute-PREROUTING - [0:0]', + ':nova-compute-POSTROUTING - [0:0]', + ':nova-postrouting-bottom - [0:0]', + '-A PREROUTING -j nova-compute-PREROUTING ', + '-A OUTPUT -j nova-compute-OUTPUT ', + '-A POSTROUTING -j nova-compute-POSTROUTING ', + '-A POSTROUTING -j nova-postrouting-bottom ', + '-A nova-postrouting-bottom -j nova-compute-SNATTING ', + '-A nova-compute-SNATTING -j nova-compute-floating-ip-snat ', + 'COMMIT', + '# Completed on Fri Feb 18 15:17:05 2011'] + + def setUp(self): + super(IptablesManagerTestCase, self).setUp() + self.manager = linux_net.IptablesManager() + + def test_filter_rules_are_wrapped(self): + current_lines = self.sample_filter + + table = self.manager.ipv4['filter'] + table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') + new_lines = self.manager._modify_rules(current_lines, table) + self.assertTrue('-A run_tests.py-FORWARD ' + '-s 1.2.3.4/5 -j DROP' in new_lines) + + table.remove_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') + new_lines = self.manager._modify_rules(current_lines, table) + self.assertTrue('-A run_tests.py-FORWARD ' + '-s 1.2.3.4/5 -j DROP' not in new_lines) + + def test_nat_rules(self): + current_lines = self.sample_nat + new_lines = self.manager._modify_rules(current_lines, + self.manager.ipv4['nat']) + + for line in [':nova-compute-OUTPUT - [0:0]', + ':nova-compute-floating-ip-snat - [0:0]', + ':nova-compute-SNATTING - [0:0]', + ':nova-compute-PREROUTING - [0:0]', + ':nova-compute-POSTROUTING - [0:0]']: + self.assertTrue(line in new_lines, "One of nova-compute's chains " + "went missing.") + + seen_lines = set() + for line in new_lines: + line = line.strip() + self.assertTrue(line not in seen_lines, + "Duplicate line: %s" % line) + seen_lines.add(line) + + last_postrouting_line = '' + + for line in new_lines: + if line.startswith('-A POSTROUTING'): + last_postrouting_line = line + + self.assertTrue('-j nova-postrouting-bottom' in last_postrouting_line, + "Last POSTROUTING rule does not jump to " + "nova-postouting-bottom: %s" % last_postrouting_line) + + for chain in ['POSTROUTING', 'PREROUTING', 'OUTPUT']: + self.assertTrue('-A %s -j run_tests.py-%s' \ + % (chain, chain) in new_lines, + "Built-in chain %s not wrapped" % (chain,)) + + def test_filter_rules(self): + current_lines = self.sample_filter + new_lines = self.manager._modify_rules(current_lines, + self.manager.ipv4['filter']) + + for line in [':nova-compute-FORWARD - [0:0]', + ':nova-compute-INPUT - [0:0]', + ':nova-compute-local - [0:0]', + ':nova-compute-OUTPUT - [0:0]']: + self.assertTrue(line in new_lines, "One of nova-compute's chains" + " went missing.") + + seen_lines = set() + for line in new_lines: + line = line.strip() + self.assertTrue(line not in seen_lines, + "Duplicate line: %s" % line) + seen_lines.add(line) + + for chain in ['FORWARD', 'OUTPUT']: + for line in new_lines: + if line.startswith('-A %s' % chain): + self.assertTrue('-j nova-filter-top' in line, + "First %s rule does not " + "jump to nova-filter-top" % chain) + break + + self.assertTrue('-A nova-filter-top ' + '-j run_tests.py-local' in new_lines, + "nova-filter-top does not jump to wrapped local chain") + + for chain in ['INPUT', 'OUTPUT', 'FORWARD']: + self.assertTrue('-A %s -j run_tests.py-%s' \ + % (chain, chain) in new_lines, + "Built-in chain %s not wrapped" % (chain,)) + + class NetworkTestCase(test.TestCase): """Test cases for network code""" def setUp(self): diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index b6888c4d2..bb279ac4b 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -155,7 +155,7 @@ class SimpleDriverTestCase(test.TestCase): def _create_instance(self, **kwargs): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id @@ -169,8 +169,6 @@ class SimpleDriverTestCase(test.TestCase): def _create_volume(self): """Create a test volume""" vol = {} - vol['image_id'] = 'ami-test' - vol['reservation_id'] = 'r-fakeres' vol['size'] = 1 vol['availability_zone'] = 'test' return db.volume_create(self.context, vol)['id'] diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index d47e6a222..3a0f58aba 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import re import os import eventlet @@ -304,16 +305,22 @@ class IptablesFirewallTestCase(test.TestCase): self.manager.delete_user(self.user) super(IptablesFirewallTestCase, self).tearDown() - in_rules = [ + in_nat_rules = [ + '# Generated by iptables-save v1.4.10 on Sat Feb 19 00:03:19 2011', + '*nat', + ':PREROUTING ACCEPT [1170:189210]', + ':INPUT ACCEPT [844:71028]', + ':OUTPUT ACCEPT [5149:405186]', + ':POSTROUTING ACCEPT [5063:386098]', + ] + + in_filter_rules = [ '# Generated by iptables-save v1.4.4 on Mon Dec 6 11:54:13 2010', '*filter', ':INPUT ACCEPT [969615:281627771]', ':FORWARD ACCEPT [0:0]', ':OUTPUT ACCEPT [915599:63811649]', ':nova-block-ipv4 - [0:0]', - '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ', - '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ', - '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ', '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ', '-A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED' ',ESTABLISHED -j ACCEPT ', @@ -325,7 +332,7 @@ class IptablesFirewallTestCase(test.TestCase): '# Completed on Mon Dec 6 11:54:13 2010', ] - in6_rules = [ + in6_filter_rules = [ '# Generated by ip6tables-save v1.4.4 on Tue Jan 18 23:47:56 2011', '*filter', ':INPUT ACCEPT [349155:75810423]', @@ -388,21 +395,31 @@ class IptablesFirewallTestCase(test.TestCase): def fake_iptables_execute(*cmd, **kwargs): process_input = kwargs.get('process_input', None) if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): - return '\n'.join(self.in6_rules), None + return '\n'.join(self.in6_filter_rules), None if cmd == ('sudo', 'iptables-save', '-t', 'filter'): - return '\n'.join(self.in_rules), None + return '\n'.join(self.in_filter_rules), None + if cmd == ('sudo', 'iptables-save', '-t', 'nat'): + return '\n'.join(self.in_nat_rules), None if cmd == ('sudo', 'iptables-restore'): - self.out_rules = process_input.split('\n') + lines = process_input.split('\n') + if '*filter' in lines: + self.out_rules = lines return '', '' if cmd == ('sudo', 'ip6tables-restore'): - self.out6_rules = process_input.split('\n') + lines = process_input.split('\n') + if '*filter' in lines: + self.out6_rules = lines return '', '' - self.fw.execute = fake_iptables_execute + print cmd, kwargs + + from nova.network import linux_net + linux_net.iptables_manager.execute = fake_iptables_execute self.fw.prepare_instance_filter(instance_ref) self.fw.apply_instance_filter(instance_ref) - in_rules = filter(lambda l: not l.startswith('#'), self.in_rules) + in_rules = filter(lambda l: not l.startswith('#'), + self.in_filter_rules) for rule in in_rules: if not 'nova' in rule: self.assertTrue(rule in self.out_rules, @@ -425,17 +442,18 @@ class IptablesFirewallTestCase(test.TestCase): self.assertTrue(security_group_chain, "The security group chain wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -j ACCEPT' % \ - security_group_chain in self.out_rules, + regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "ICMP acceptance rule wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -m icmp --icmp-type ' - '8 -j ACCEPT' % security_group_chain in self.out_rules, + regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -m icmp ' + '--icmp-type 8 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "ICMP Echo Request acceptance rule wasn't added") - self.assertTrue('-A %s -p tcp -s 192.168.10.0/24 -m multiport ' - '--dports 80:81 -j ACCEPT' % security_group_chain \ - in self.out_rules, + regex = re.compile('-A .* -p tcp -s 192.168.10.0/24 -m multiport ' + '--dports 80:81 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "TCP port 80/81 acceptance rule wasn't added") db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index b40ca004b..f698c85b5 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -99,7 +99,7 @@ class VolumeTestCase(test.TestCase): def test_run_attach_detach_volume(self): """Make sure volume can be attached and detached from instance.""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = 'fake' diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index c26dc8639..cd125a301 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -18,6 +18,7 @@ Test suite for XenAPI """ +import functools import stubout from nova import db @@ -41,6 +42,21 @@ from nova.tests.glance import stubs as glance_stubs FLAGS = flags.FLAGS +def stub_vm_utils_with_vdi_attached_here(function, should_return=True): + """ + vm_utils.with_vdi_attached_here needs to be stubbed out because it + calls down to the filesystem to attach a vdi. This provides a + decorator to handle that. + """ + @functools.wraps(function) + def decorated_function(self, *args, **kwargs): + orig_with_vdi_attached_here = vm_utils.with_vdi_attached_here + vm_utils.with_vdi_attached_here = lambda *x: should_return + function(self, *args, **kwargs) + vm_utils.with_vdi_attached_here = orig_with_vdi_attached_here + return decorated_function + + class XenAPIVolumeTestCase(test.TestCase): """ Unit tests for Volume operations @@ -62,6 +78,7 @@ class XenAPIVolumeTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux' } def _create_volume(self, size='0'): @@ -219,7 +236,7 @@ class XenAPIVMTestCase(test.TestCase): check() - def check_vm_record(self, conn): + def create_vm_record(self, conn, os_type): instances = conn.list_instances() self.assertEquals(instances, [1]) @@ -231,28 +248,63 @@ class XenAPIVMTestCase(test.TestCase): in xenapi_fake.get_all_records('VM').iteritems() if not rec['is_control_domain']] vm = vms[0] + self.vm_info = vm_info + self.vm = vm + def check_vm_record(self, conn): # Check that m1.large above turned into the right thing. instance_type = db.instance_type_get_by_name(conn, 'm1.large') mem_kib = long(instance_type['memory_mb']) << 10 mem_bytes = str(mem_kib << 10) vcpus = instance_type['vcpus'] - self.assertEquals(vm_info['max_mem'], mem_kib) - self.assertEquals(vm_info['mem'], mem_kib) - self.assertEquals(vm['memory_static_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_min'], mem_bytes) - self.assertEquals(vm['VCPUs_max'], str(vcpus)) - self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) + self.assertEquals(self.vm_info['max_mem'], mem_kib) + self.assertEquals(self.vm_info['mem'], mem_kib) + self.assertEquals(self.vm['memory_static_max'], mem_bytes) + self.assertEquals(self.vm['memory_dynamic_max'], mem_bytes) + self.assertEquals(self.vm['memory_dynamic_min'], mem_bytes) + self.assertEquals(self.vm['VCPUs_max'], str(vcpus)) + self.assertEquals(self.vm['VCPUs_at_startup'], str(vcpus)) # Check that the VM is running according to Nova - self.assertEquals(vm_info['state'], power_state.RUNNING) + self.assertEquals(self.vm_info['state'], power_state.RUNNING) # Check that the VM is running according to XenAPI. - self.assertEquals(vm['power_state'], 'Running') + self.assertEquals(self.vm['power_state'], 'Running') + + def check_vm_params_for_windows(self): + self.assertEquals(self.vm['platform']['nx'], 'true') + self.assertEquals(self.vm['HVM_boot_params'], {'order': 'dc'}) + self.assertEquals(self.vm['HVM_boot_policy'], 'BIOS order') + + # check that these are not set + self.assertEquals(self.vm['PV_args'], '') + self.assertEquals(self.vm['PV_bootloader'], '') + self.assertEquals(self.vm['PV_kernel'], '') + self.assertEquals(self.vm['PV_ramdisk'], '') + + def check_vm_params_for_linux(self): + self.assertEquals(self.vm['platform']['nx'], 'false') + self.assertEquals(self.vm['PV_args'], 'clocksource=jiffies') + self.assertEquals(self.vm['PV_bootloader'], 'pygrub') + + # check that these are not set + self.assertEquals(self.vm['PV_kernel'], '') + self.assertEquals(self.vm['PV_ramdisk'], '') + self.assertEquals(self.vm['HVM_boot_params'], {}) + self.assertEquals(self.vm['HVM_boot_policy'], '') + + def check_vm_params_for_linux_with_external_kernel(self): + self.assertEquals(self.vm['platform']['nx'], 'false') + self.assertEquals(self.vm['PV_args'], 'root=/dev/xvda1') + self.assertNotEquals(self.vm['PV_kernel'], '') + self.assertNotEquals(self.vm['PV_ramdisk'], '') + + # check that these are not set + self.assertEquals(self.vm['HVM_boot_params'], {}) + self.assertEquals(self.vm['HVM_boot_policy'], '') def _test_spawn(self, image_id, kernel_id, ramdisk_id, - instance_type="m1.large"): + instance_type="m1.large", os_type="linux"): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) values = {'name': 1, 'id': 1, @@ -263,10 +315,12 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': ramdisk_id, 'instance_type': instance_type, 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': os_type } conn = xenapi_conn.get_connection(False) instance = db.instance_create(values) conn.spawn(instance) + self.create_vm_record(conn, os_type) self.check_vm_record(conn) def test_spawn_not_enough_memory(self): @@ -283,24 +337,37 @@ class XenAPIVMTestCase(test.TestCase): FLAGS.xenapi_image_service = 'objectstore' self._test_spawn(1, 2, 3) + @stub_vm_utils_with_vdi_attached_here def test_spawn_raw_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_RAW, None, None) + self.check_vm_params_for_linux() - def test_spawn_vhd_glance(self): + def test_spawn_vhd_glance_linux(self): FLAGS.xenapi_image_service = 'glance' - self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None) + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, + os_type="linux") + self.check_vm_params_for_linux() + + def test_spawn_vhd_glance_windows(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, + os_type="windows") + self.check_vm_params_for_windows() def test_spawn_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, glance_stubs.FakeGlance.IMAGE_RAMDISK) + self.check_vm_params_for_linux_with_external_kernel() def tearDown(self): super(XenAPIVMTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + self.vm_info = None + self.vm = None self.stubs.UnsetAll() def _create_instance(self): @@ -314,7 +381,8 @@ class XenAPIVMTestCase(test.TestCase): 'kernel_id': 2, 'ramdisk_id': 3, 'instance_type': 'm1.large', - 'mac_address': 'aa:bb:cc:dd:ee:ff'} + 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux'} instance = db.instance_create(values) self.conn.spawn(instance) return instance @@ -372,6 +440,7 @@ class XenAPIMigrateInstance(test.TestCase): 'ramdisk_id': None, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux' } stubs.stub_out_migration_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, @@ -410,6 +479,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance = FakeInstance() self.fake_instance.id = 42 + self.fake_instance.os_type = 'linux' def assert_disk_type(self, disk_type): dt = vm_utils.VMHelper.determine_disk_image_type( diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py new file mode 100644 index 000000000..5a52a0506 --- /dev/null +++ b/nova/tests/test_zones.py @@ -0,0 +1,172 @@ +# Copyright 2010 United States Government as represented by the +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For ZoneManager +""" + +import datetime +import mox +import novaclient + +from nova import context +from nova import db +from nova import flags +from nova import service +from nova import test +from nova import rpc +from nova import utils +from nova.auth import manager as auth_manager +from nova.scheduler import zone_manager + +FLAGS = flags.FLAGS + + +class FakeZone: + """Represents a fake zone from the db""" + def __init__(self, *args, **kwargs): + for k, v in kwargs.iteritems(): + setattr(self, k, v) + + +def exploding_novaclient(zone): + """Used when we want to simulate a novaclient call failing.""" + raise Exception("kaboom") + + +class ZoneManagerTestCase(test.TestCase): + """Test case for zone manager""" + def test_ping(self): + zm = zone_manager.ZoneManager() + self.mox.StubOutWithMock(zm, '_refresh_from_db') + self.mox.StubOutWithMock(zm, '_poll_zones') + zm._refresh_from_db(mox.IgnoreArg()) + zm._poll_zones(mox.IgnoreArg()) + + self.mox.ReplayAll() + zm.ping(None) + self.mox.VerifyAll() + + def test_refresh_from_db_new(self): + zm = zone_manager.ZoneManager() + + self.mox.StubOutWithMock(db, 'zone_get_all') + db.zone_get_all(mox.IgnoreArg()).AndReturn([ + FakeZone(id=1, api_url='http://foo.com', username='user1', + password='pass1'), + ]) + + self.assertEquals(len(zm.zone_states), 0) + + self.mox.ReplayAll() + zm._refresh_from_db(None) + self.mox.VerifyAll() + + self.assertEquals(len(zm.zone_states), 1) + self.assertEquals(zm.zone_states[1].username, 'user1') + + def test_refresh_from_db_replace_existing(self): + zm = zone_manager.ZoneManager() + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com', + username='user1', password='pass1')) + zm.zone_states[1] = zone_state + + self.mox.StubOutWithMock(db, 'zone_get_all') + db.zone_get_all(mox.IgnoreArg()).AndReturn([ + FakeZone(id=1, api_url='http://foo.com', username='user2', + password='pass2'), + ]) + + self.assertEquals(len(zm.zone_states), 1) + + self.mox.ReplayAll() + zm._refresh_from_db(None) + self.mox.VerifyAll() + + self.assertEquals(len(zm.zone_states), 1) + self.assertEquals(zm.zone_states[1].username, 'user2') + + def test_refresh_from_db_missing(self): + zm = zone_manager.ZoneManager() + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com', + username='user1', password='pass1')) + zm.zone_states[1] = zone_state + + self.mox.StubOutWithMock(db, 'zone_get_all') + db.zone_get_all(mox.IgnoreArg()).AndReturn([]) + + self.assertEquals(len(zm.zone_states), 1) + + self.mox.ReplayAll() + zm._refresh_from_db(None) + self.mox.VerifyAll() + + self.assertEquals(len(zm.zone_states), 0) + + def test_refresh_from_db_add_and_delete(self): + zm = zone_manager.ZoneManager() + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com', + username='user1', password='pass1')) + zm.zone_states[1] = zone_state + + self.mox.StubOutWithMock(db, 'zone_get_all') + + db.zone_get_all(mox.IgnoreArg()).AndReturn([ + FakeZone(id=2, api_url='http://foo.com', username='user2', + password='pass2'), + ]) + self.assertEquals(len(zm.zone_states), 1) + + self.mox.ReplayAll() + zm._refresh_from_db(None) + self.mox.VerifyAll() + + self.assertEquals(len(zm.zone_states), 1) + self.assertEquals(zm.zone_states[2].username, 'user2') + + def test_poll_zone(self): + self.mox.StubOutWithMock(zone_manager, '_call_novaclient') + zone_manager._call_novaclient(mox.IgnoreArg()).AndReturn( + dict(name='zohan', capabilities='hairdresser')) + + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=2, + api_url='http://foo.com', username='user2', + password='pass2')) + zone_state.attempt = 1 + + self.mox.ReplayAll() + zone_manager._poll_zone(zone_state) + self.mox.VerifyAll() + self.assertEquals(zone_state.attempt, 0) + self.assertEquals(zone_state.name, 'zohan') + + def test_poll_zone_fails(self): + self.stubs.Set(zone_manager, "_call_novaclient", exploding_novaclient) + + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=2, + api_url='http://foo.com', username='user2', + password='pass2')) + zone_state.attempt = FLAGS.zone_failures_to_offline - 1 + + self.mox.ReplayAll() + zone_manager._poll_zone(zone_state) + self.mox.VerifyAll() + self.assertEquals(zone_state.attempt, 3) + self.assertFalse(zone_state.is_active) + self.assertEquals(zone_state.name, None) diff --git a/nova/utils.py b/nova/utils.py index 0466fecf4..87e726394 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -139,34 +139,44 @@ def execute(*cmd, **kwargs): stdin = kwargs.get('stdin', subprocess.PIPE) stdout = kwargs.get('stdout', subprocess.PIPE) stderr = kwargs.get('stderr', subprocess.PIPE) + attempts = kwargs.get('attempts', 1) cmd = map(str, cmd) - LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) - env = os.environ.copy() - if addl_env: - env.update(addl_env) - obj = subprocess.Popen(cmd, stdin=stdin, - stdout=stdout, stderr=stderr, env=env) - result = None - if process_input != None: - result = obj.communicate(process_input) - else: - result = obj.communicate() - obj.stdin.close() - if obj.returncode: - LOG.debug(_("Result was %s") % obj.returncode) - if type(check_exit_code) == types.IntType \ - and obj.returncode != check_exit_code: - (stdout, stderr) = result - raise ProcessExecutionError(exit_code=obj.returncode, - stdout=stdout, - stderr=stderr, - cmd=' '.join(cmd)) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one - greenthread.sleep(0) - return result + while attempts > 0: + attempts -= 1 + try: + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) + env = os.environ.copy() + if addl_env: + env.update(addl_env) + obj = subprocess.Popen(cmd, stdin=stdin, + stdout=stdout, stderr=stderr, env=env) + result = None + if process_input != None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + obj.stdin.close() + if obj.returncode: + LOG.debug(_("Result was %s") % obj.returncode) + if type(check_exit_code) == types.IntType \ + and obj.returncode != check_exit_code: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=obj.returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + LOG.debug(_("%r failed. Retrying."), cmd) + greenthread.sleep(random.randint(20, 200) / 100.0) def ssh_execute(ssh, cmd, process_input=None, diff --git a/nova/wsgi.py b/nova/wsgi.py index 2d18da8fb..ba0819466 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -436,7 +436,8 @@ class Serializer(object): try: return handlers[content_type] except Exception: - raise exception.InvalidContentType() + raise exception.InvalidContentType(_("Invalid content type %s" + % content_type)) def _from_json(self, datastring): return utils.loads(datastring)