diff --git a/.mailmap b/.mailmap index 7e031fc7..3f0238ee 100644 --- a/.mailmap +++ b/.mailmap @@ -29,6 +29,7 @@ + @@ -36,6 +37,7 @@ + @@ -44,5 +46,4 @@ - diff --git a/Authors b/Authors index 60e1d2da..50f4680a 100644 --- a/Authors +++ b/Authors @@ -1,4 +1,5 @@ Alex Meade +Andrey Brindeyev Andy Smith Andy Southgate Anne Gentle @@ -16,6 +17,7 @@ Christian Berendt Chuck Short Cory Wright Dan Prince +Dave Walker David Pravec Dean Troyer Devin Carlen @@ -44,12 +46,14 @@ Josh Kearney Josh Kleinpeter Joshua McKenty Justin Santa Barbara +Justin Shepherd Kei Masumoto Ken Pepple Kevin Bringard Kevin L. Mitchell Koji Iida Lorin Hochstein +Lvov Maxim Mark Washenberger Masanori Itoh Matt Dietz @@ -62,6 +66,7 @@ Nachi Ueno Naveed Massjouni Nirmal Ranganathan Paul Voccio +Renuka Apte Ricardo Carrillo Cruz Rick Clark Rick Harris @@ -78,7 +83,7 @@ Trey Morris Tushar Patil Vasiliy Shlykov Vishvananda Ishaya -William Wolf +William Wolf Yoshiaki Tamura Youcef Laribi Yuriy Taraday diff --git a/MANIFEST.in b/MANIFEST.in index e7a6e7da..4e145de7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -35,6 +35,7 @@ include nova/tests/bundle/1mb.manifest.xml include nova/tests/bundle/1mb.no_kernel_or_ramdisk.manifest.xml include nova/tests/bundle/1mb.part.0 include nova/tests/bundle/1mb.part.1 +include nova/tests/public_key/* include nova/tests/db/nova.austin.sqlite include plugins/xenapi/README include plugins/xenapi/etc/xapi.d/plugins/objectstore diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index f42dfd6b..5926b97d 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -108,6 +108,13 @@ def main(): interface = os.environ.get('DNSMASQ_INTERFACE', FLAGS.dnsmasq_interface) if int(os.environ.get('TESTING', '0')): from nova.tests import fake_flags + + #if FLAGS.fake_rabbit: + # LOG.debug(_("leasing ip")) + # network_manager = utils.import_object(FLAGS.network_manager) + ## reload(fake_flags) + # from nova.tests import fake_flags + action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] diff --git a/bin/nova-manage b/bin/nova-manage index 2f6af6e2..e3ed7b9d 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -362,27 +362,47 @@ class ProjectCommands(object): def add(self, project_id, user_id): """Adds user to project arguments: project_id user_id""" - self.manager.add_to_project(user_id, project_id) + try: + self.manager.add_to_project(user_id, project_id) + except exception.UserNotFound as ex: + print ex + raise def create(self, name, project_manager, description=None): """Creates a new project arguments: name project_manager [description]""" - self.manager.create_project(name, project_manager, description) + try: + self.manager.create_project(name, project_manager, description) + except exception.UserNotFound as ex: + print ex + raise def modify(self, name, project_manager, description=None): """Modifies a project arguments: name project_manager [description]""" - self.manager.modify_project(name, project_manager, description) + try: + self.manager.modify_project(name, project_manager, description) + except exception.UserNotFound as ex: + print ex + raise def delete(self, name): """Deletes an existing project arguments: name""" - self.manager.delete_project(name) + try: + self.manager.delete_project(name) + except exception.ProjectNotFound as ex: + print ex + raise def environment(self, project_id, user_id, filename='novarc'): """Exports environment variables to an sourcable file arguments: project_id user_id [filename='novarc]""" - rc = self.manager.get_environment_rc(user_id, project_id) + try: + rc = self.manager.get_environment_rc(user_id, project_id) + except (exception.UserNotFound, exception.ProjectNotFound) as ex: + print ex + raise with open(filename, 'w') as f: f.write(rc) @@ -397,11 +417,10 @@ class ProjectCommands(object): arguments: project_id [key] [value]""" ctxt = context.get_admin_context() if key: - quo = {'project_id': project_id, key: value} try: - db.quota_update(ctxt, project_id, quo) - except exception.NotFound: - db.quota_create(ctxt, quo) + db.quota_update(ctxt, project_id, key, value) + except exception.ProjectQuotaNotFound: + db.quota_create(ctxt, project_id, key, value) project_quota = quota.get_quota(ctxt, project_id) for key, value in project_quota.iteritems(): print '%s: %s' % (key, value) @@ -409,7 +428,11 @@ class ProjectCommands(object): def remove(self, project_id, user_id): """Removes user from project arguments: project_id user_id""" - self.manager.remove_from_project(user_id, project_id) + try: + self.manager.remove_from_project(user_id, project_id) + except (exception.UserNotFound, exception.ProjectNotFound) as ex: + print ex + raise def scrub(self, project_id): """Deletes data associated with project @@ -428,6 +451,9 @@ class ProjectCommands(object): zip_file = self.manager.get_credentials(user_id, project_id) with open(filename, 'w') as f: f.write(zip_file) + except (exception.UserNotFound, exception.ProjectNotFound) as ex: + print ex + raise except db.api.NoMoreNetworks: print _('No more networks available. If this is a new ' 'installation, you need\nto call something like this:\n\n' @@ -523,8 +549,10 @@ class NetworkCommands(object): [network_size=FLAG], [vlan_start=FLAG], [vpn_start=FLAG], [fixed_range_v6=FLAG]""" if not fixed_range: - raise TypeError(_('Fixed range in the form of 10.0.0.0/8 is ' - 'required to create networks.')) + msg = _('Fixed range in the form of 10.0.0.0/8 is ' + 'required to create networks.') + print msg + raise TypeError(msg) if not num_networks: num_networks = FLAGS.num_networks if not network_size: @@ -536,14 +564,18 @@ class NetworkCommands(object): if not fixed_range_v6: fixed_range_v6 = FLAGS.fixed_range_v6 net_manager = utils.import_object(FLAGS.network_manager) - net_manager.create_networks(context.get_admin_context(), - cidr=fixed_range, - num_networks=int(num_networks), - network_size=int(network_size), - vlan_start=int(vlan_start), - vpn_start=int(vpn_start), - cidr_v6=fixed_range_v6, - label=label) + try: + net_manager.create_networks(context.get_admin_context(), + cidr=fixed_range, + num_networks=int(num_networks), + network_size=int(network_size), + vlan_start=int(vlan_start), + vpn_start=int(vpn_start), + cidr_v6=fixed_range_v6, + label=label) + except ValueError, e: + print e + raise e def list(self): """List all created networks""" @@ -972,7 +1004,7 @@ class ImageCommands(object): try: internal_id = ec2utils.ec2_id_to_id(old_image_id) image = self.image_service.show(context, internal_id) - except exception.NotFound: + except (exception.InvalidEc2Id, exception.ImageNotFound): image = self.image_service.show_by_name(context, old_image_id) return image['id'] diff --git a/bin/stack b/bin/stack index d84a82e2..a1c6d134 100755 --- a/bin/stack +++ b/bin/stack @@ -65,7 +65,7 @@ def format_help(d): indent = MAX_INDENT - 6 out = [] - for k, v in d.iteritems(): + for k, v in sorted(d.iteritems()): if (len(k) + 6) > MAX_INDENT: out.extend([' %s' % k]) initial_indent = ' ' * (indent + 6) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 98d5f6eb..07235a2a 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -305,9 +305,9 @@ class AuthManager(object): if check_type == 's3': sign = signer.Signer(user.secret.encode()) expected_signature = sign.s3_authorization(headers, verb, path) - LOG.debug('user.secret: %s', user.secret) - LOG.debug('expected_signature: %s', expected_signature) - LOG.debug('signature: %s', signature) + LOG.debug(_('user.secret: %s'), user.secret) + LOG.debug(_('expected_signature: %s'), expected_signature) + LOG.debug(_('signature: %s'), signature) if signature != expected_signature: LOG.audit(_("Invalid signature for user %s"), user.name) raise exception.InvalidSignature(signature=signature, @@ -317,10 +317,20 @@ class AuthManager(object): # secret isn't unicode expected_signature = signer.Signer(user.secret.encode()).generate( params, verb, server_string, path) - LOG.debug('user.secret: %s', user.secret) - LOG.debug('expected_signature: %s', expected_signature) - LOG.debug('signature: %s', signature) + LOG.debug(_('user.secret: %s'), user.secret) + LOG.debug(_('expected_signature: %s'), expected_signature) + LOG.debug(_('signature: %s'), signature) if signature != expected_signature: + (addr_str, port_str) = utils.parse_server_string(server_string) + # If the given server_string contains port num, try without it. + if port_str != '': + host_only_signature = signer.Signer( + user.secret.encode()).generate(params, verb, + addr_str, path) + LOG.debug(_('host_only_signature: %s'), + host_only_signature) + if signature == host_only_signature: + return (user, project) LOG.audit(_("Invalid signature for user %s"), user.name) raise exception.InvalidSignature(signature=signature, user=user) diff --git a/nova/flags.py b/nova/flags.py index 51979364..9eaac559 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -110,7 +110,7 @@ class FlagValues(gflags.FlagValues): return name in self.__dict__['__dirty'] def ClearDirty(self): - self.__dict__['__is_dirty'] = [] + self.__dict__['__dirty'] = [] def WasAlreadyParsed(self): return self.__dict__['__was_already_parsed'] @@ -119,11 +119,12 @@ class FlagValues(gflags.FlagValues): if '__stored_argv' not in self.__dict__: return new_flags = FlagValues(self) - for k in self.__dict__['__dirty']: + for k in self.FlagDict().iterkeys(): new_flags[k] = gflags.FlagValues.__getitem__(self, k) + new_flags.Reset() new_flags(self.__dict__['__stored_argv']) - for k in self.__dict__['__dirty']: + for k in new_flags.FlagDict().iterkeys(): setattr(self, k, getattr(new_flags, k)) self.ClearDirty() @@ -369,6 +370,9 @@ DEFINE_string('host', socket.gethostname(), DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') +DEFINE_string('notification_driver', + 'nova.notifier.no_op_notifier', + 'Default driver for sending notifications') DEFINE_list('memcached_servers', None, 'Memcached servers or None for in process cache.') diff --git a/nova/notifier/__init__.py b/nova/notifier/__init__.py new file mode 100644 index 00000000..482d54e4 --- /dev/null +++ b/nova/notifier/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/nova/notifier/api.py b/nova/notifier/api.py new file mode 100644 index 00000000..a3e7a039 --- /dev/null +++ b/nova/notifier/api.py @@ -0,0 +1,83 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.import datetime + +import datetime +import uuid + +from nova import flags +from nova import utils + + +FLAGS = flags.FLAGS + +flags.DEFINE_string('default_notification_level', 'INFO', + 'Default notification level for outgoing notifications') + +WARN = 'WARN' +INFO = 'INFO' +ERROR = 'ERROR' +CRITICAL = 'CRITICAL' +DEBUG = 'DEBUG' + +log_levels = (DEBUG, WARN, INFO, ERROR, CRITICAL) + + +class BadPriorityException(Exception): + pass + + +def notify(publisher_id, event_type, priority, payload): + """ + Sends a notification using the specified driver + + Notify parameters: + + publisher_id - the source worker_type.host of the message + event_type - the literal type of event (ex. Instance Creation) + priority - patterned after the enumeration of Python logging levels in + the set (DEBUG, WARN, INFO, ERROR, CRITICAL) + payload - A python dictionary of attributes + + Outgoing message format includes the above parameters, and appends the + following: + + message_id - a UUID representing the id for this notification + timestamp - the GMT timestamp the notification was sent at + + The composite message will be constructed as a dictionary of the above + attributes, which will then be sent via the transport mechanism defined + by the driver. + + Message example: + + {'message_id': str(uuid.uuid4()), + 'publisher_id': 'compute.host1', + 'timestamp': datetime.datetime.utcnow(), + 'priority': 'WARN', + 'event_type': 'compute.create_instance', + 'payload': {'instance_id': 12, ... }} + + """ + if priority not in log_levels: + raise BadPriorityException( + _('%s not in valid priorities' % priority)) + driver = utils.import_object(FLAGS.notification_driver) + msg = dict(message_id=str(uuid.uuid4()), + publisher_id=publisher_id, + event_type=event_type, + priority=priority, + payload=payload, + timestamp=str(datetime.datetime.utcnow())) + driver.notify(msg) diff --git a/nova/notifier/log_notifier.py b/nova/notifier/log_notifier.py new file mode 100644 index 00000000..25dfc693 --- /dev/null +++ b/nova/notifier/log_notifier.py @@ -0,0 +1,34 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from nova import flags +from nova import log as logging + + +FLAGS = flags.FLAGS + + +def notify(message): + """Notifies the recipient of the desired event given the model. + Log notifications using nova's default logging system""" + + priority = message.get('priority', + FLAGS.default_notification_level) + priority = priority.lower() + logger = logging.getLogger( + 'nova.notification.%s' % message['event_type']) + getattr(logger, priority)(json.dumps(message)) diff --git a/nova/tests/real_flags.py b/nova/notifier/no_op_notifier.py similarity index 64% rename from nova/tests/real_flags.py rename to nova/notifier/no_op_notifier.py index 71da0499..02971050 100644 --- a/nova/tests/real_flags.py +++ b/nova/notifier/no_op_notifier.py @@ -1,7 +1,4 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -16,11 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -from nova import flags -FLAGS = flags.FLAGS - -FLAGS.connection_type = 'libvirt' -FLAGS.fake_rabbit = False -FLAGS.fake_network = False -FLAGS.verbose = False +def notify(message): + """Notifies the recipient of the desired event given the model""" + pass diff --git a/nova/notifier/rabbit_notifier.py b/nova/notifier/rabbit_notifier.py new file mode 100644 index 00000000..d46670b5 --- /dev/null +++ b/nova/notifier/rabbit_notifier.py @@ -0,0 +1,36 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import nova.context + +from nova import flags +from nova import rpc + + +FLAGS = flags.FLAGS + +flags.DEFINE_string('notification_topic', 'notifications', + 'RabbitMQ topic used for Nova notifications') + + +def notify(message): + """Sends a notification to the RabbitMQ""" + context = nova.context.get_admin_context() + priority = message.get('priority', + FLAGS.default_notification_level) + priority = priority.lower() + topic = '%s.%s' % (FLAGS.notification_topic, priority) + rpc.cast(context, topic, message) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 816ae551..55f8e0a6 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -81,6 +81,12 @@ def get_zone_capabilities(context): return _call_scheduler('get_zone_capabilities', context=context) +def select(context, specs=None): + """Returns a list of hosts.""" + return _call_scheduler('select', context=context, + params={"specs": specs}) + + def update_service_capabilities(context, service_name, host, capabilities): """Send an update to all the scheduler services informing them of the capabilities of this service.""" @@ -105,6 +111,45 @@ def _process(func, zone): return func(nova, zone) +def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs): + """Returns a list of (zone, call_result) objects.""" + if not isinstance(errors_to_ignore, (list, tuple)): + # This will also handle the default None + errors_to_ignore = [errors_to_ignore] + + pool = greenpool.GreenPool() + results = [] + for zone in db.zone_get_all(context): + try: + nova = novaclient.OpenStack(zone.username, zone.password, + zone.api_url) + nova.authenticate() + except novaclient.exceptions.BadRequest, e: + url = zone.api_url + LOG.warn(_("Failed request to zone; URL=%(url)s: %(e)s") + % locals()) + #TODO (dabo) - add logic for failure counts per zone, + # with escalation after a given number of failures. + continue + zone_method = getattr(nova.zones, method) + + def _error_trap(*args, **kwargs): + try: + return zone_method(*args, **kwargs) + except Exception as e: + if type(e) in errors_to_ignore: + return None + # TODO (dabo) - want to be able to re-raise here. + # Returning a string now; raising was causing issues. + # raise e + return "ERROR", "%s" % e + + res = pool.spawn(_error_trap, *args, **kwargs) + results.append((zone, res)) + pool.waitall() + return [(zone.id, res.wait()) for zone, res in results] + + def child_zone_helper(zone_list, func): """Fire off a command to each zone in the list. The return is [novaclient return objects] from each child zone. diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index ed8e65c7..483f3225 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -96,8 +96,8 @@ class FlavorFilter(HostFilter): selected_hosts = [] for host, services in zone_manager.service_states.iteritems(): capabilities = services.get('compute', {}) - host_ram_mb = capabilities['host_memory']['free'] - disk_bytes = capabilities['disk']['available'] + host_ram_mb = capabilities['host_memory_free'] + disk_bytes = capabilities['disk_available'] if host_ram_mb >= instance_type['memory_mb'] and \ disk_bytes >= instance_type['local_gb']: selected_hosts.append((host, capabilities)) @@ -106,16 +106,16 @@ class FlavorFilter(HostFilter): #host entries (currently) are like: # {'host_name-description': 'Default install of XenServer', # 'host_hostname': 'xs-mini', -# 'host_memory': {'total': 8244539392, -# 'overhead': 184225792, -# 'free': 3868327936, -# 'free-computed': 3840843776}, +# 'host_memory_total': 8244539392, +# 'host_memory_overhead': 184225792, +# 'host_memory_free': 3868327936, +# 'host_memory_free_computed': 3840843776}, # 'host_other-config': {}, # 'host_ip_address': '192.168.1.109', # 'host_cpu_info': {}, -# 'disk': {'available': 32954957824, -# 'total': 50394562560, -# 'used': 17439604736}, +# 'disk_available': 32954957824, +# 'disk_total': 50394562560, +# 'disk_used': 17439604736}, # 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', # 'host_name-label': 'xs-mini'} @@ -221,8 +221,8 @@ class JsonFilter(HostFilter): required_ram = instance_type['memory_mb'] required_disk = instance_type['local_gb'] query = ['and', - ['>=', '$compute.host_memory.free', required_ram], - ['>=', '$compute.disk.available', required_disk] + ['>=', '$compute.host_memory_free', required_ram], + ['>=', '$compute.disk_available', required_disk] ] return (self._full_name(), json.dumps(query)) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py new file mode 100644 index 00000000..b3d230bd --- /dev/null +++ b/nova/scheduler/zone_aware_scheduler.py @@ -0,0 +1,119 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +The Zone Aware Scheduler is a base class Scheduler for creating instances +across zones. There are two expansion points to this class for: +1. Assigning Weights to hosts for requested instances +2. Filtering Hosts based on required instance capabilities +""" + +import operator + +from nova import log as logging +from nova.scheduler import api +from nova.scheduler import driver + +LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler') + + +class ZoneAwareScheduler(driver.Scheduler): + """Base class for creating Zone Aware Schedulers.""" + + def _call_zone_method(self, context, method, specs): + """Call novaclient zone method. Broken out for testing.""" + return api.call_zone_method(context, method, specs=specs) + + def schedule_run_instance(self, context, topic='compute', specs={}, + *args, **kwargs): + """This method is called from nova.compute.api to provision + an instance. However we need to look at the parameters being + passed in to see if this is a request to: + 1. Create a Build Plan and then provision, or + 2. Use the Build Plan information in the request parameters + to simply create the instance (either in this zone or + a child zone).""" + + if 'blob' in specs: + return self.provision_instance(context, topic, specs) + + # Create build plan and provision ... + build_plan = self.select(context, specs) + for item in build_plan: + self.provision_instance(context, topic, item) + + def provision_instance(context, topic, item): + """Create the requested instance in this Zone or a child zone.""" + pass + + def select(self, context, *args, **kwargs): + """Select returns a list of weights and zone/host information + corresponding to the best hosts to service the request. Any + child zone information has been encrypted so as not to reveal + anything about the children.""" + return self._schedule(context, "compute", *args, **kwargs) + + def schedule(self, context, topic, *args, **kwargs): + """The schedule() contract requires we return the one + best-suited host for this request. + """ + res = self._schedule(context, topic, *args, **kwargs) + # TODO(sirp): should this be a host object rather than a weight-dict? + if not res: + raise driver.NoValidHost(_('No hosts were available')) + return res[0] + + def _schedule(self, context, topic, *args, **kwargs): + """Returns a list of hosts that meet the required specs, + ordered by their fitness. + """ + + #TODO(sandy): extract these from args. + num_instances = 1 + specs = {} + + # Filter local hosts based on requirements ... + host_list = self.filter_hosts(num_instances, specs) + + # then weigh the selected hosts. + # weighted = [{weight=weight, name=hostname}, ...] + weighted = self.weigh_hosts(num_instances, specs, host_list) + + # Next, tack on the best weights from the child zones ... + child_results = self._call_zone_method(context, "select", + specs=specs) + for child_zone, result in child_results: + for weighting in result: + # Remember the child_zone so we can get back to + # it later if needed. This implicitly builds a zone + # path structure. + host_dict = { + "weight": weighting["weight"], + "child_zone": child_zone, + "child_blob": weighting["blob"]} + weighted.append(host_dict) + + weighted.sort(key=operator.itemgetter('weight')) + return weighted + + def filter_hosts(self, num, specs): + """Derived classes must override this method and return + a list of hosts in [(hostname, capability_dict)] format.""" + raise NotImplemented() + + def weigh_hosts(self, num, specs, hosts): + """Derived classes must override this method and return + a lists of hosts in [{weight, hostname}] format.""" + raise NotImplemented() diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 5d7ca98b..ecefc464 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -21,24 +21,24 @@ from nova import flags FLAGS = flags.FLAGS flags.DECLARE('volume_driver', 'nova.volume.manager') -FLAGS.volume_driver = 'nova.volume.driver.FakeISCSIDriver' -FLAGS.connection_type = 'fake' -FLAGS.fake_rabbit = True +FLAGS['volume_driver'].SetDefault('nova.volume.driver.FakeISCSIDriver') +FLAGS['connection_type'].SetDefault('fake') +FLAGS['fake_rabbit'].SetDefault(True) flags.DECLARE('auth_driver', 'nova.auth.manager') -FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' +FLAGS['auth_driver'].SetDefault('nova.auth.dbdriver.DbDriver') flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') flags.DECLARE('fake_network', 'nova.network.manager') -FLAGS.network_size = 8 -FLAGS.num_networks = 2 -FLAGS.fake_network = True -FLAGS.image_service = 'nova.image.local.LocalImageService' +FLAGS['network_size'].SetDefault(8) +FLAGS['num_networks'].SetDefault(2) +FLAGS['fake_network'].SetDefault(True) +FLAGS['image_service'].SetDefault('nova.image.local.LocalImageService') flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') flags.DECLARE('iscsi_num_targets', 'nova.volume.driver') -FLAGS.num_shelves = 2 -FLAGS.blades_per_shelf = 4 -FLAGS.iscsi_num_targets = 8 -FLAGS.verbose = True -FLAGS.sqlite_db = "tests.sqlite" -FLAGS.use_ipv6 = True +FLAGS['num_shelves'].SetDefault(2) +FLAGS['blades_per_shelf'].SetDefault(4) +FLAGS['iscsi_num_targets'].SetDefault(8) +FLAGS['verbose'].SetDefault(True) +FLAGS['sqlite_db'].SetDefault("tests.sqlite") +FLAGS['use_ipv6'].SetDefault(True) diff --git a/nova/tests/public_key/dummy.fingerprint b/nova/tests/public_key/dummy.fingerprint new file mode 100644 index 00000000..715bca27 --- /dev/null +++ b/nova/tests/public_key/dummy.fingerprint @@ -0,0 +1 @@ +1c:87:d1:d9:32:fd:62:3c:78:2b:c0:ad:c0:15:88:df diff --git a/nova/tests/public_key/dummy.pub b/nova/tests/public_key/dummy.pub new file mode 100644 index 00000000..d4cf2bc0 --- /dev/null +++ b/nova/tests/public_key/dummy.pub @@ -0,0 +1 @@ +ssh-dss AAAAB3NzaC1kc3MAAACBAMGJlY9XEIm2X234pdO5yFWMp2JuOQx8U0E815IVXhmKxYCBK9ZakgZOIQmPbXoGYyV+mziDPp6HJ0wKYLQxkwLEFr51fAZjWQvRss0SinURRuLkockDfGFtD4pYJthekr/rlqMKlBSDUSpGq8jUWW60UJ18FGooFpxR7ESqQRx/AAAAFQC96LRglaUeeP+E8U/yblEJocuiWwAAAIA3XiMR8Skiz/0aBm5K50SeQznQuMJTyzt9S9uaz5QZWiFu69hOyGSFGw8fqgxEkXFJIuHobQQpGYQubLW0NdaYRqyE/Vud3JUJUb8Texld6dz8vGemyB5d1YvtSeHIo8/BGv2msOqR3u5AZTaGCBD9DhpSGOKHEdNjTtvpPd8S8gAAAIBociGZ5jf09iHLVENhyXujJbxfGRPsyNTyARJfCOGl0oFV6hEzcQyw8U/ePwjgvjc2UizMWLl8tsb2FXKHRdc2v+ND3Us+XqKQ33X3ADP4FZ/+Oj213gMyhCmvFTP0u5FmHog9My4CB7YcIWRuUR42WlhQ2IfPvKwUoTk3R+T6Og== www-data@mk diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index fa0e5659..7c0331ef 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -28,10 +28,12 @@ import StringIO import webob from nova import context +from nova import exception from nova import test from nova.api import ec2 -from nova.api.ec2 import cloud from nova.api.ec2 import apirequest +from nova.api.ec2 import cloud +from nova.api.ec2 import ec2utils from nova.auth import manager @@ -101,6 +103,21 @@ class XmlConversionTestCase(test.TestCase): self.assertEqual(conv('-0'), 0) +class Ec2utilsTestCase(test.TestCase): + def test_ec2_id_to_id(self): + self.assertEqual(ec2utils.ec2_id_to_id('i-0000001e'), 30) + self.assertEqual(ec2utils.ec2_id_to_id('ami-1d'), 29) + + def test_bad_ec2_id(self): + self.assertRaises(exception.InvalidEc2Id, + ec2utils.ec2_id_to_id, + 'badone') + + def test_id_to_ec2_id(self): + self.assertEqual(ec2utils.id_to_ec2_id(30), 'i-0000001e') + self.assertEqual(ec2utils.id_to_ec2_id(29, 'ami-%08x'), 'ami-0000001d') + + class ApiEc2TestCase(test.TestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): @@ -207,6 +224,29 @@ class ApiEc2TestCase(test.TestCase): self.manager.delete_project(project) self.manager.delete_user(user) + def test_create_duplicate_key_pair(self): + """Test that, after successfully generating a keypair, + requesting a second keypair with the same name fails sanely""" + self.expect_http() + self.mox.ReplayAll() + keyname = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ + for x in range(random.randint(4, 8))) + user = self.manager.create_user('fake', 'fake', 'fake') + project = self.manager.create_project('fake', 'fake', 'fake') + # NOTE(vish): create depends on pool, so call helper directly + self.ec2.create_key_pair('test') + + try: + self.ec2.create_key_pair('test') + except EC2ResponseError, e: + if e.code == 'KeyPairExists': + pass + else: + self.fail("Unexpected EC2ResponseError: %s " + "(expected KeyPairExists)" % e.code) + else: + self.fail('Exception not raised.') + def test_get_all_security_groups(self): """Test that we can retrieve security groups""" self.expect_http() diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index f8a1b156..f02dd94b 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -101,9 +101,43 @@ class _AuthManagerBaseTestCase(test.TestCase): self.assertEqual('private-party', u.access) def test_004_signature_is_valid(self): - #self.assertTrue(self.manager.authenticate(**boto.generate_url ...? )) - pass - #raise NotImplementedError + with user_generator(self.manager, name='admin', secret='admin', + access='admin'): + with project_generator(self.manager, name="admin", + manager_user='admin'): + accesskey = 'admin:admin' + expected_result = (self.manager.get_user('admin'), + self.manager.get_project('admin')) + # captured sig and query string using boto 1.9b/euca2ools 1.2 + sig = 'd67Wzd9Bwz8xid9QU+lzWXcF2Y3tRicYABPJgrqfrwM=' + auth_params = {'AWSAccessKeyId': 'admin:admin', + 'Action': 'DescribeAvailabilityZones', + 'SignatureMethod': 'HmacSHA256', + 'SignatureVersion': '2', + 'Timestamp': '2011-04-22T11:29:29', + 'Version': '2009-11-30'} + self.assertTrue(expected_result, self.manager.authenticate( + accesskey, + sig, + auth_params, + 'GET', + '127.0.0.1:8773', + '/services/Cloud/')) + # captured sig and query string using RightAWS 1.10.0 + sig = 'ECYLU6xdFG0ZqRVhQybPJQNJ5W4B9n8fGs6+/fuGD2c=' + auth_params = {'AWSAccessKeyId': 'admin:admin', + 'Action': 'DescribeAvailabilityZones', + 'SignatureMethod': 'HmacSHA256', + 'SignatureVersion': '2', + 'Timestamp': '2011-04-22T11:29:49.000Z', + 'Version': '2008-12-01'} + self.assertTrue(expected_result, self.manager.authenticate( + accesskey, + sig, + auth_params, + 'GET', + '127.0.0.1', + '/services/Cloud')) def test_005_can_get_credentials(self): return diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index f271c03f..54c0454d 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -279,6 +279,26 @@ class CloudTestCase(test.TestCase): user_group=['all']) self.assertEqual(True, result['is_public']) + def test_deregister_image(self): + deregister_image = self.cloud.deregister_image + + def fake_delete(self, context, id): + return None + + self.stubs.Set(local.LocalImageService, 'delete', fake_delete) + # valid image + result = deregister_image(self.context, 'ami-00000001') + self.assertEqual(result['imageId'], 'ami-00000001') + # invalid image + self.stubs.UnsetAll() + + def fake_detail_empty(self, context): + return [] + + self.stubs.Set(local.LocalImageService, 'detail', fake_detail_empty) + self.assertRaises(exception.ImageNotFound, deregister_image, + self.context, 'ami-bad001') + def test_console_output(self): instance_type = FLAGS.default_instance_type max_count = 1 @@ -334,45 +354,40 @@ class CloudTestCase(test.TestCase): self.assertTrue(filter(lambda k: k['keyName'] == 'test1', keys)) self.assertTrue(filter(lambda k: k['keyName'] == 'test2', keys)) + def test_import_public_key(self): + # test when user provides all values + result1 = self.cloud.import_public_key(self.context, + 'testimportkey1', + 'mytestpubkey', + 'mytestfprint') + self.assertTrue(result1) + keydata = db.key_pair_get(self.context, + self.context.user.id, + 'testimportkey1') + self.assertEqual('mytestpubkey', keydata['public_key']) + self.assertEqual('mytestfprint', keydata['fingerprint']) + # test when user omits fingerprint + pubkey_path = os.path.join(os.path.dirname(__file__), 'public_key') + f = open(pubkey_path + '/dummy.pub', 'r') + dummypub = f.readline().rstrip() + f.close + f = open(pubkey_path + '/dummy.fingerprint', 'r') + dummyfprint = f.readline().rstrip() + f.close + result2 = self.cloud.import_public_key(self.context, + 'testimportkey2', + dummypub) + self.assertTrue(result2) + keydata = db.key_pair_get(self.context, + self.context.user.id, + 'testimportkey2') + self.assertEqual(dummypub, keydata['public_key']) + self.assertEqual(dummyfprint, keydata['fingerprint']) + def test_delete_key_pair(self): self._create_key('test') self.cloud.delete_key_pair(self.context, 'test') - def test_run_instances(self): - if FLAGS.connection_type == 'fake': - LOG.debug(_("Can't test instances without a real virtual env.")) - return - image_id = FLAGS.default_image - instance_type = FLAGS.default_instance_type - max_count = 1 - kwargs = {'image_id': image_id, - 'instance_type': instance_type, - 'max_count': max_count} - rv = self.cloud.run_instances(self.context, **kwargs) - # TODO: check for proper response - instance_id = rv['reservationSet'][0].keys()[0] - instance = rv['reservationSet'][0][instance_id][0] - LOG.debug(_("Need to watch instance %s until it's running..."), - instance['instance_id']) - while True: - greenthread.sleep(1) - info = self.cloud._get_instance(instance['instance_id']) - LOG.debug(info['state']) - if info['state'] == power_state.RUNNING: - break - self.assert_(rv) - - if FLAGS.connection_type != 'fake': - time.sleep(45) # Should use boto for polling here - for reservations in rv['reservationSet']: - # for res_id in reservations.keys(): - # LOG.debug(reservations[res_id]) - # for instance in reservations[res_id]: - for instance in reservations[reservations.keys()[0]]: - instance_id = instance['instance_id'] - LOG.debug(_("Terminating instance %s"), instance_id) - rv = self.compute.terminate_instance(instance_id) - def test_terminate_instances(self): inst1 = db.instance_create(self.context, {'reservation_id': 'a', 'image_id': 1, diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 39311079..9170837b 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -21,6 +21,7 @@ Tests For Compute import datetime import mox +import stubout from nova import compute from nova import context @@ -52,6 +53,10 @@ class FakeTime(object): self.counter += t +def nop_report_driver_status(self): + pass + + class ComputeTestCase(test.TestCase): """Test case for compute""" def setUp(self): @@ -329,6 +334,28 @@ class ComputeTestCase(test.TestCase): self.compute.terminate_instance(self.context, instance_id) + def test_finish_resize(self): + """Contrived test to ensure finish_resize doesn't raise anything""" + + def fake(*args, **kwargs): + pass + + self.stubs.Set(self.compute.driver, 'finish_resize', fake) + context = self.context.elevated() + instance_id = self._create_instance() + self.compute.prep_resize(context, instance_id, 1) + migration_ref = db.migration_get_by_instance_and_status(context, + instance_id, 'pre-migrating') + try: + self.compute.finish_resize(context, instance_id, + int(migration_ref['id']), {}) + except KeyError, e: + # Only catch key errors. We want other reasons for the test to + # fail to actually error out so we don't obscure anything + self.fail() + + self.compute.terminate_instance(self.context, instance_id) + def test_resize_instance(self): """Ensure instance can be migrated/resized""" instance_id = self._create_instance() @@ -649,6 +676,10 @@ class ComputeTestCase(test.TestCase): def test_run_kill_vm(self): """Detect when a vm is terminated behind the scenes""" + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(compute_manager.ComputeManager, + '_report_driver_status', nop_report_driver_status) + instance_id = self._create_instance() self.compute.run_instance(self.context, instance_id) diff --git a/nova/tests/test_flags.py b/nova/tests/test_flags.py index 707300fc..05319d91 100644 --- a/nova/tests/test_flags.py +++ b/nova/tests/test_flags.py @@ -91,6 +91,20 @@ class FlagsTestCase(test.TestCase): self.assert_('runtime_answer' in self.global_FLAGS) self.assertEqual(self.global_FLAGS.runtime_answer, 60) + def test_long_vs_short_flags(self): + flags.DEFINE_string('duplicate_answer_long', 'val', 'desc', + flag_values=self.global_FLAGS) + argv = ['flags_test', '--duplicate_answer=60', 'extra_arg'] + args = self.global_FLAGS(argv) + + self.assert_('duplicate_answer' not in self.global_FLAGS) + self.assert_(self.global_FLAGS.duplicate_answer_long, 60) + + flags.DEFINE_integer('duplicate_answer', 60, 'desc', + flag_values=self.global_FLAGS) + self.assertEqual(self.global_FLAGS.duplicate_answer, 60) + self.assertEqual(self.global_FLAGS.duplicate_answer_long, 'val') + def test_flag_leak_left(self): self.assertEqual(FLAGS.flags_unittest, 'foo') FLAGS.flags_unittest = 'bar' diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py index 31e40ae1..c029d41e 100644 --- a/nova/tests/test_host_filter.py +++ b/nova/tests/test_host_filter.py @@ -43,16 +43,16 @@ class HostFilterTestCase(test.TestCase): # which means ... don't go above 10 hosts. return {'host_name-description': 'XenServer %s' % multiplier, 'host_hostname': 'xs-%s' % multiplier, - 'host_memory': {'total': 100, - 'overhead': 10, - 'free': 10 + multiplier * 10, - 'free-computed': 10 + multiplier * 10}, + 'host_memory_total': 100, + 'host_memory_overhead': 10, + 'host_memory_free': 10 + multiplier * 10, + 'host_memory_free-computed': 10 + multiplier * 10, 'host_other-config': {}, 'host_ip_address': '192.168.1.%d' % (100 + multiplier), 'host_cpu_info': {}, - 'disk': {'available': 100 + multiplier * 100, - 'total': 1000, - 'used': 0}, + 'disk_available': 100 + multiplier * 100, + 'disk_total': 1000, + 'disk_used': 0, 'host_uuid': 'xxx-%d' % multiplier, 'host_name-label': 'xs-%s' % multiplier} @@ -131,12 +131,12 @@ class HostFilterTestCase(test.TestCase): raw = ['or', ['and', - ['<', '$compute.host_memory.free', 30], - ['<', '$compute.disk.available', 300] + ['<', '$compute.host_memory_free', 30], + ['<', '$compute.disk_available', 300] ], ['and', - ['>', '$compute.host_memory.free', 70], - ['>', '$compute.disk.available', 700] + ['>', '$compute.host_memory_free', 70], + ['>', '$compute.disk_available', 700] ] ] cooked = json.dumps(raw) @@ -149,7 +149,7 @@ class HostFilterTestCase(test.TestCase): self.assertEquals('host%02d' % index, host) raw = ['not', - ['=', '$compute.host_memory.free', 30], + ['=', '$compute.host_memory_free', 30], ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -160,7 +160,7 @@ class HostFilterTestCase(test.TestCase): for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] + raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) diff --git a/nova/tests/test_notifier.py b/nova/tests/test_notifier.py new file mode 100644 index 00000000..b6b0fcc6 --- /dev/null +++ b/nova/tests/test_notifier.py @@ -0,0 +1,117 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import nova + +from nova import context +from nova import flags +from nova import rpc +import nova.notifier.api +from nova.notifier.api import notify +from nova.notifier import no_op_notifier +from nova.notifier import rabbit_notifier +from nova import test + +import stubout + + +class NotifierTestCase(test.TestCase): + """Test case for notifications""" + def setUp(self): + super(NotifierTestCase, self).setUp() + self.stubs = stubout.StubOutForTesting() + + def tearDown(self): + self.stubs.UnsetAll() + super(NotifierTestCase, self).tearDown() + + def test_send_notification(self): + self.notify_called = False + + def mock_notify(cls, *args): + self.notify_called = True + + self.stubs.Set(nova.notifier.no_op_notifier, 'notify', + mock_notify) + + class Mock(object): + pass + notify('publisher_id', 'event_type', + nova.notifier.api.WARN, dict(a=3)) + self.assertEqual(self.notify_called, True) + + def test_verify_message_format(self): + """A test to ensure changing the message format is prohibitively + annoying""" + + def message_assert(message): + fields = [('publisher_id', 'publisher_id'), + ('event_type', 'event_type'), + ('priority', 'WARN'), + ('payload', dict(a=3))] + for k, v in fields: + self.assertEqual(message[k], v) + self.assertTrue(len(message['message_id']) > 0) + self.assertTrue(len(message['timestamp']) > 0) + + self.stubs.Set(nova.notifier.no_op_notifier, 'notify', + message_assert) + notify('publisher_id', 'event_type', + nova.notifier.api.WARN, dict(a=3)) + + def test_send_rabbit_notification(self): + self.stubs.Set(nova.flags.FLAGS, 'notification_driver', + 'nova.notifier.rabbit_notifier') + self.mock_cast = False + + def mock_cast(cls, *args): + self.mock_cast = True + + class Mock(object): + pass + + self.stubs.Set(nova.rpc, 'cast', mock_cast) + notify('publisher_id', 'event_type', + nova.notifier.api.WARN, dict(a=3)) + + self.assertEqual(self.mock_cast, True) + + def test_invalid_priority(self): + def mock_cast(cls, *args): + pass + + class Mock(object): + pass + + self.stubs.Set(nova.rpc, 'cast', mock_cast) + self.assertRaises(nova.notifier.api.BadPriorityException, + notify, 'publisher_id', + 'event_type', 'not a priority', dict(a=3)) + + def test_rabbit_priority_queue(self): + self.stubs.Set(nova.flags.FLAGS, 'notification_driver', + 'nova.notifier.rabbit_notifier') + self.stubs.Set(nova.flags.FLAGS, 'notification_topic', + 'testnotify') + + self.test_topic = None + + def mock_cast(context, topic, msg): + self.test_topic = topic + + self.stubs.Set(nova.rpc, 'cast', mock_cast) + notify('publisher_id', + 'event_type', 'DEBUG', dict(a=3)) + self.assertEqual(self.test_topic, 'testnotify.debug') diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 968ef9d6..54b3f80f 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -912,7 +912,8 @@ class SimpleDriverTestCase(test.TestCase): class FakeZone(object): - def __init__(self, api_url, username, password): + def __init__(self, id, api_url, username, password): + self.id = id self.api_url = api_url self.username = username self.password = password @@ -920,7 +921,7 @@ class FakeZone(object): def zone_get_all(context): return [ - FakeZone('http://example.com', 'bob', 'xxx'), + FakeZone(1, 'http://example.com', 'bob', 'xxx'), ] @@ -1037,7 +1038,7 @@ class FakeNovaClient(object): class DynamicNovaClientTest(test.TestCase): def test_issue_novaclient_command_found(self): - zone = FakeZone('http://example.com', 'bob', 'xxx') + zone = FakeZone(1, 'http://example.com', 'bob', 'xxx') self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeServerCollection()), zone, "servers", "get", 100).a, 10) @@ -1051,7 +1052,7 @@ class DynamicNovaClientTest(test.TestCase): zone, "servers", "pause", 100), None) def test_issue_novaclient_command_not_found(self): - zone = FakeZone('http://example.com', 'bob', 'xxx') + zone = FakeZone(1, 'http://example.com', 'bob', 'xxx') self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), zone, "servers", "get", 100), None) @@ -1063,3 +1064,55 @@ class DynamicNovaClientTest(test.TestCase): self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), zone, "servers", "any", "name"), None) + + +class FakeZonesProxy(object): + def do_something(*args, **kwargs): + return 42 + + def raises_exception(*args, **kwargs): + raise Exception('testing') + + +class FakeNovaClientOpenStack(object): + def __init__(self, *args, **kwargs): + self.zones = FakeZonesProxy() + + def authenticate(self): + pass + + +class CallZoneMethodTest(test.TestCase): + def setUp(self): + super(CallZoneMethodTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(db, 'zone_get_all', zone_get_all) + self.stubs.Set(novaclient, 'OpenStack', FakeNovaClientOpenStack) + + def tearDown(self): + self.stubs.UnsetAll() + super(CallZoneMethodTest, self).tearDown() + + def test_call_zone_method(self): + context = {} + method = 'do_something' + results = api.call_zone_method(context, method) + expected = [(1, 42)] + self.assertEqual(expected, results) + + def test_call_zone_method_not_present(self): + context = {} + method = 'not_present' + self.assertRaises(AttributeError, api.call_zone_method, + context, method) + + def test_call_zone_method_generates_exception(self): + context = {} + method = 'raises_exception' + results = api.call_zone_method(context, method) + + # FIXME(sirp): for now the _error_trap code is catching errors and + # converting them to a ("ERROR", "string") tuples. The code (and this + # test) should eventually handle real exceptions. + expected = [(1, ('ERROR', 'testing'))] + self.assertEqual(expected, results) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 1311ba36..1bec9cac 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -642,7 +642,7 @@ class LibvirtConnTestCase(test.TestCase): try: conn.spawn(instance, network_info) except Exception, e: - count = (0 <= e.message.find('Unexpected method call')) + count = (0 <= str(e.message).find('Unexpected method call')) self.assertTrue(count) @@ -849,7 +849,7 @@ class IptablesFirewallTestCase(test.TestCase): self.assertEquals(len(rulesv4), 2) self.assertEquals(len(rulesv6), 0) - def multinic_iptables_test(self): + def test_multinic_iptables(self): ipv4_rules_per_network = 2 ipv6_rules_per_network = 3 networks_count = 5 @@ -869,6 +869,16 @@ class IptablesFirewallTestCase(test.TestCase): self.assertEquals(ipv6_network_rules, ipv6_rules_per_network * networks_count) + def test_do_refresh_security_group_rules(self): + instance_ref = self._create_instance_ref() + self.mox.StubOutWithMock(self.fw, + 'add_filters_for_instance', + use_mock_anything=True) + self.fw.add_filters_for_instance(instance_ref, mox.IgnoreArg()) + self.fw.instances[instance_ref['id']] = instance_ref + self.mox.ReplayAll() + self.fw.do_refresh_security_group_rules("fake") + class NWFilterTestCase(test.TestCase): def setUp(self): diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 375480a2..be1e3569 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -16,7 +16,9 @@ """Test suite for XenAPI.""" +import eventlet import functools +import json import os import re import stubout @@ -197,6 +199,28 @@ class XenAPIVMTestCase(test.TestCase): self.context = context.RequestContext('fake', 'fake', False) self.conn = xenapi_conn.get_connection(False) + def test_parallel_builds(self): + stubs.stubout_loopingcall_delay(self.stubs) + + def _do_build(id, proj, user, *args): + values = { + 'id': id, + 'project_id': proj, + 'user_id': user, + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type_id': '3', # m1.large + 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux'} + instance = db.instance_create(self.context, values) + self.conn.spawn(instance) + + gt1 = eventlet.spawn(_do_build, 1, self.project.id, self.user.id) + gt2 = eventlet.spawn(_do_build, 2, self.project.id, self.user.id) + gt1.wait() + gt2.wait() + def test_list_instances_0(self): instances = self.conn.list_instances() self.assertEquals(instances, []) @@ -665,3 +689,52 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_VHD self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_VHD) + + +class FakeXenApi(object): + """Fake XenApi for testing HostState.""" + + class FakeSR(object): + def get_record(self, ref): + return {'virtual_allocation': 10000, + 'physical_utilisation': 20000} + + SR = FakeSR() + + +class FakeSession(object): + """Fake Session class for HostState testing.""" + + def async_call_plugin(self, *args): + return None + + def wait_for_task(self, *args): + vm = {'total': 10, + 'overhead': 20, + 'free': 30, + 'free-computed': 40} + return json.dumps({'host_memory': vm}) + + def get_xenapi(self): + return FakeXenApi() + + +class HostStateTestCase(test.TestCase): + """Tests HostState, which holds metrics from XenServer that get + reported back to the Schedulers.""" + + def _fake_safe_find_sr(self, session): + """None SR ref since we're ignoring it in FakeSR.""" + return None + + def test_host_state(self): + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(vm_utils, 'safe_find_sr', self._fake_safe_find_sr) + host_state = xenapi_conn.HostState(FakeSession()) + stats = host_state._stats + self.assertEquals(stats['disk_total'], 10000) + self.assertEquals(stats['disk_used'], 20000) + self.assertEquals(stats['host_memory_total'], 10) + self.assertEquals(stats['host_memory_overhead'], 20) + self.assertEquals(stats['host_memory_free'], 30) + self.assertEquals(stats['host_memory_free_computed'], 40) diff --git a/nova/tests/test_zone_aware_scheduler.py b/nova/tests/test_zone_aware_scheduler.py new file mode 100644 index 00000000..fdcde34c --- /dev/null +++ b/nova/tests/test_zone_aware_scheduler.py @@ -0,0 +1,119 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Zone Aware Scheduler. +""" + +from nova import test +from nova.scheduler import driver +from nova.scheduler import zone_aware_scheduler +from nova.scheduler import zone_manager + + +class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler): + def filter_hosts(self, num, specs): + # NOTE(sirp): this is returning [(hostname, services)] + return self.zone_manager.service_states.items() + + def weigh_hosts(self, num, specs, hosts): + fake_weight = 99 + weighted = [] + for hostname, caps in hosts: + weighted.append(dict(weight=fake_weight, name=hostname)) + return weighted + + +class FakeZoneManager(zone_manager.ZoneManager): + def __init__(self): + self.service_states = { + 'host1': { + 'compute': {'ram': 1000} + }, + 'host2': { + 'compute': {'ram': 2000} + }, + 'host3': { + 'compute': {'ram': 3000} + } + } + + +class FakeEmptyZoneManager(zone_manager.ZoneManager): + def __init__(self): + self.service_states = {} + + +def fake_empty_call_zone_method(context, method, specs): + return [] + + +def fake_call_zone_method(context, method, specs): + return [ + ('zone1', [ + dict(weight=1, blob='AAAAAAA'), + dict(weight=111, blob='BBBBBBB'), + dict(weight=112, blob='CCCCCCC'), + dict(weight=113, blob='DDDDDDD'), + ]), + ('zone2', [ + dict(weight=120, blob='EEEEEEE'), + dict(weight=2, blob='FFFFFFF'), + dict(weight=122, blob='GGGGGGG'), + dict(weight=123, blob='HHHHHHH'), + ]), + ('zone3', [ + dict(weight=130, blob='IIIIIII'), + dict(weight=131, blob='JJJJJJJ'), + dict(weight=132, blob='KKKKKKK'), + dict(weight=3, blob='LLLLLLL'), + ]), + ] + + +class ZoneAwareSchedulerTestCase(test.TestCase): + """Test case for Zone Aware Scheduler.""" + + def test_zone_aware_scheduler(self): + """ + Create a nested set of FakeZones, ensure that a select call returns the + appropriate build plan. + """ + sched = FakeZoneAwareScheduler() + self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method) + + zm = FakeZoneManager() + sched.set_zone_manager(zm) + + fake_context = {} + build_plan = sched.select(fake_context, {}) + + self.assertEqual(15, len(build_plan)) + + hostnames = [plan_item['name'] + for plan_item in build_plan if 'name' in plan_item] + self.assertEqual(3, len(hostnames)) + + def test_empty_zone_aware_scheduler(self): + """ + Ensure empty hosts & child_zones result in NoValidHosts exception. + """ + sched = FakeZoneAwareScheduler() + self.stubs.Set(sched, '_call_zone_method', fake_empty_call_zone_method) + + zm = FakeEmptyZoneManager() + sched.set_zone_manager(zm) + + fake_context = {} + self.assertRaises(driver.NoValidHost, sched.schedule, fake_context, {}) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 205f6c90..4833ccb0 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -16,6 +16,7 @@ """Stubouts, mocks and fixtures for the test suite""" +import eventlet from nova.virt import xenapi_conn from nova.virt.xenapi import fake from nova.virt.xenapi import volume_utils @@ -28,29 +29,6 @@ def stubout_instance_snapshot(stubs): @classmethod def fake_fetch_image(cls, session, instance_id, image, user, project, type): - # Stubout wait_for_task - def fake_wait_for_task(self, task, id): - class FakeEvent: - - def send(self, value): - self.rv = value - - def wait(self): - return self.rv - - done = FakeEvent() - self._poll_task(id, task, done) - rv = done.wait() - return rv - - def fake_loop(self): - pass - - stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', - fake_wait_for_task) - - stubs.Set(xenapi_conn.XenAPISession, '_stop_loop', fake_loop) - from nova.virt.xenapi.fake import create_vdi name_label = "instance-%s" % instance_id #TODO: create fake SR record @@ -63,11 +41,6 @@ def stubout_instance_snapshot(stubs): stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image) - def fake_parse_xmlrpc_value(val): - return val - - stubs.Set(xenapi_conn, '_parse_xmlrpc_value', fake_parse_xmlrpc_value) - def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, original_parent_uuid): from nova.virt.xenapi.fake import create_vdi @@ -144,6 +117,16 @@ def stubout_loopingcall_start(stubs): stubs.Set(utils.LoopingCall, 'start', fake_start) +def stubout_loopingcall_delay(stubs): + def fake_start(self, interval, now=True): + self._running = True + eventlet.sleep(1) + self.f(*self.args, **self.kw) + # This would fail before parallel xenapi calls were fixed + assert self._running == False + stubs.Set(utils.LoopingCall, 'start', fake_start) + + class FakeSessionForVMTests(fake.SessionBase): """ Stubs out a XenAPISession for VM tests """ def __init__(self, uri):