diff --git a/Authors b/Authors index 02fe46c7..86467992 100644 --- a/Authors +++ b/Authors @@ -101,6 +101,7 @@ Stephanie Reese Thierry Carrez Todd Willey Trey Morris +Troy Toman Tushar Patil Vasiliy Shlykov Vishvananda Ishaya diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 2329581a..0a789b4b 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -24,7 +24,6 @@ from eventlet import greenthread from eventlet.green import urllib2 import exceptions -import gettext import os import sys import time @@ -38,11 +37,11 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -gettext.install('nova', unicode=1) from nova import flags from nova import log as logging from nova import rpc +from nova import service from nova import utils from nova import wsgi @@ -141,5 +140,5 @@ if __name__ == '__main__': acp = AjaxConsoleProxy() acp.register_listeners() server = wsgi.Server("AJAX Console Proxy", acp, port=acp_port) - server.start() - server.wait() + service.serve(server) + service.wait() diff --git a/bin/nova-api b/bin/nova-api index fe8e8336..38e2624d 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -19,12 +19,14 @@ """Starter script for Nova API. -Starts both the EC2 and OpenStack APIs in separate processes. +Starts both the EC2 and OpenStack APIs in separate greenthreads. """ +import eventlet +eventlet.monkey_patch() + import os -import signal import sys @@ -33,32 +35,18 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath( if os.path.exists(os.path.join(possible_topdir, "nova", "__init__.py")): sys.path.insert(0, possible_topdir) -import nova.service -import nova.utils from nova import flags - - -FLAGS = flags.FLAGS - - -def main(): - """Launch EC2 and OSAPI services.""" - nova.utils.Bootstrapper.bootstrap_binary(sys.argv) - - launcher = nova.service.Launcher() - - for api in FLAGS.enabled_apis: - service = nova.service.WSGIService(api) - launcher.launch_service(service) - - signal.signal(signal.SIGTERM, lambda *_: launcher.stop()) - - try: - launcher.wait() - except KeyboardInterrupt: - launcher.stop() - +from nova import log as logging +from nova import service +from nova import utils if __name__ == '__main__': - sys.exit(main()) + utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() + servers = [] + for api in flags.FLAGS.enabled_apis: + servers.append(service.WSGIService(api)) + service.serve(*servers) + service.wait() diff --git a/bin/nova-api-ec2 b/bin/nova-api-ec2 new file mode 100755 index 00000000..df50f713 --- /dev/null +++ b/bin/nova-api-ec2 @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starter script for Nova EC2 API.""" + +import eventlet +eventlet.monkey_patch() + +import os +import sys + + +possible_topdir = os.path.normpath(os.path.join(os.path.abspath( + sys.argv[0]), os.pardir, os.pardir)) +if os.path.exists(os.path.join(possible_topdir, "nova", "__init__.py")): + sys.path.insert(0, possible_topdir) + + +from nova import flags +from nova import log as logging +from nova import service +from nova import utils + +if __name__ == '__main__': + utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() + server = service.WSGIService('ec2') + service.serve(server) + service.wait() diff --git a/bin/nova-api-os b/bin/nova-api-os new file mode 100755 index 00000000..374e850e --- /dev/null +++ b/bin/nova-api-os @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starter script for Nova OS API.""" + +import eventlet +eventlet.monkey_patch() + +import os +import sys + + +possible_topdir = os.path.normpath(os.path.join(os.path.abspath( + sys.argv[0]), os.pardir, os.pardir)) +if os.path.exists(os.path.join(possible_topdir, "nova", "__init__.py")): + sys.path.insert(0, possible_topdir) + + +from nova import flags +from nova import log as logging +from nova import service +from nova import utils + +if __name__ == '__main__': + utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() + server = service.WSGIService('osapi') + service.serve(server) + service.wait() diff --git a/bin/nova-compute b/bin/nova-compute index cd7c78de..5239fae7 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -22,7 +22,6 @@ import eventlet eventlet.monkey_patch() -import gettext import os import sys @@ -34,7 +33,6 @@ POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'nova', '__init__.py')): sys.path.insert(0, POSSIBLE_TOPDIR) -gettext.install('nova', unicode=1) from nova import flags from nova import log as logging @@ -45,5 +43,6 @@ if __name__ == '__main__': utils.default_flagfile() flags.FLAGS(sys.argv) logging.setup() - service.serve() + server = service.Service.create(binary='nova-compute') + service.serve(server) service.wait() diff --git a/bin/nova-console b/bin/nova-console index 40608b99..22f6ef17 100755 --- a/bin/nova-console +++ b/bin/nova-console @@ -21,7 +21,6 @@ import eventlet eventlet.monkey_patch() -import gettext import os import sys @@ -33,7 +32,6 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -gettext.install('nova', unicode=1) from nova import flags from nova import log as logging @@ -44,5 +42,6 @@ if __name__ == '__main__': utils.default_flagfile() flags.FLAGS(sys.argv) logging.setup() - service.serve() + server = service.Service.create(binary='nova-console') + service.serve(server) service.wait() diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index a47ea7a7..1c9ae951 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -52,7 +52,7 @@ flags.DECLARE('update_dhcp_on_disassociate', 'nova.network.manager') LOG = logging.getLogger('nova.dhcpbridge') -def add_lease(mac, ip_address, _interface): +def add_lease(mac, ip_address): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: LOG.debug(_("leasing ip")) @@ -66,13 +66,13 @@ def add_lease(mac, ip_address, _interface): "args": {"address": ip_address}}) -def old_lease(mac, ip_address, interface): +def old_lease(mac, ip_address): """Update just as add lease.""" LOG.debug(_("Adopted old lease or got a change of mac")) - add_lease(mac, ip_address, interface) + add_lease(mac, ip_address) -def del_lease(mac, ip_address, _interface): +def del_lease(mac, ip_address): """Called when a lease expires.""" if FLAGS.fake_rabbit: LOG.debug(_("releasing ip")) @@ -99,8 +99,6 @@ def main(): utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) logging.setup() - # check ENV first so we don't break any older deploys - network_id = int(os.environ.get('NETWORK_ID')) if int(os.environ.get('TESTING', '0')): from nova.tests import fake_flags @@ -115,11 +113,19 @@ def main(): if action in ['add', 'del', 'old']: mac = argv[2] ip = argv[3] - msg = _("Called %(action)s for mac %(mac)s with ip %(ip)s" - " on interface %(interface)s") % locals() + msg = _("Called '%(action)s' for mac '%(mac)s' with ip '%(ip)s'") % \ + {"action": action, + "mac": mac, + "ip": ip} LOG.debug(msg) - globals()[action + '_lease'](mac, ip, interface) + globals()[action + '_lease'](mac, ip) else: + try: + network_id = int(os.environ.get('NETWORK_ID')) + except TypeError: + LOG.error(_("Environment variable 'NETWORK_ID' must be set.")) + sys.exit(1) + print init_leases(network_id) if __name__ == "__main__": diff --git a/bin/nova-direct-api b/bin/nova-direct-api index c6cf9b2f..106e89ba 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -20,7 +20,9 @@ """Starter script for Nova Direct API.""" -import gettext +import eventlet +eventlet.monkey_patch() + import os import sys @@ -32,12 +34,12 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -gettext.install('nova', unicode=1) from nova import compute from nova import flags from nova import log as logging from nova import network +from nova import service from nova import utils from nova import volume from nova import wsgi @@ -97,5 +99,6 @@ if __name__ == '__main__': with_auth, host=FLAGS.direct_host, port=FLAGS.direct_port) - server.start() - server.wait() + + service.serve(server) + service.wait() diff --git a/bin/nova-network b/bin/nova-network index 101761ef..57759d30 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -22,7 +22,6 @@ import eventlet eventlet.monkey_patch() -import gettext import os import sys @@ -34,7 +33,6 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -gettext.install('nova', unicode=1) from nova import flags from nova import log as logging @@ -45,5 +43,6 @@ if __name__ == '__main__': utils.default_flagfile() flags.FLAGS(sys.argv) logging.setup() - service.serve() + server = service.Service.create(binary='nova-network') + service.serve(server) service.wait() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 4d5aec44..c7a76e12 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -17,11 +17,11 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Daemon for nova objectstore. Supports S3 API. -""" +"""Daemon for nova objectstore. Supports S3 API.""" + +import eventlet +eventlet.monkey_patch() -import gettext import os import sys @@ -33,10 +33,10 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -gettext.install('nova', unicode=1) from nova import flags from nova import log as logging +from nova import service from nova import utils from nova import wsgi from nova.objectstore import s3server @@ -54,5 +54,5 @@ if __name__ == '__main__': router, port=FLAGS.s3_port, host=FLAGS.s3_host) - server.start() - server.wait() + service.serve(server) + service.wait() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 0c205a80..2e168cbc 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -22,7 +22,6 @@ import eventlet eventlet.monkey_patch() -import gettext import os import sys @@ -34,7 +33,6 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -gettext.install('nova', unicode=1) from nova import flags from nova import log as logging @@ -45,5 +43,6 @@ if __name__ == '__main__': utils.default_flagfile() flags.FLAGS(sys.argv) logging.setup() - service.serve() + server = service.Service.create(binary='nova-scheduler') + service.serve(server) service.wait() diff --git a/bin/nova-vncproxy b/bin/nova-vncproxy index bdbb30a7..dc08e243 100755 --- a/bin/nova-vncproxy +++ b/bin/nova-vncproxy @@ -19,7 +19,8 @@ """VNC Console Proxy Server.""" import eventlet -import gettext +eventlet.monkey_patch() + import os import sys @@ -29,7 +30,6 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -gettext.install('nova', unicode=1) from nova import flags from nova import log as logging @@ -41,7 +41,7 @@ from nova.vnc import auth from nova.vnc import proxy -LOG = logging.getLogger('nova.vnc-proxy') +LOG = logging.getLogger('nova.vncproxy') FLAGS = flags.FLAGS @@ -81,7 +81,7 @@ if __name__ == "__main__": FLAGS(sys.argv) logging.setup() - LOG.audit(_("Starting nova-vnc-proxy node (version %s)"), + LOG.audit(_("Starting nova-vncproxy node (version %s)"), version.version_string_with_vcs()) if not (os.path.exists(FLAGS.vncproxy_wwwroot) and @@ -107,13 +107,10 @@ if __name__ == "__main__": else: with_auth = auth.VNCNovaAuthMiddleware(with_logging) - service.serve() - server = wsgi.Server("VNC Proxy", with_auth, host=FLAGS.vncproxy_host, port=FLAGS.vncproxy_port) - server.start() server.start_tcp(handle_flash_socket_policy, 843, host=FLAGS.vncproxy_host) - - server.wait() + service.serve(server) + service.wait() diff --git a/bin/nova-volume b/bin/nova-volume index 8dcdbc50..5405aebb 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -22,7 +22,6 @@ import eventlet eventlet.monkey_patch() -import gettext import os import sys @@ -34,7 +33,6 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) -gettext.install('nova', unicode=1) from nova import flags from nova import log as logging @@ -45,5 +43,6 @@ if __name__ == '__main__': utils.default_flagfile() flags.FLAGS(sys.argv) logging.setup() - service.serve() + server = service.Service.create(binary='nova-volume') + service.serve(server) service.wait() diff --git a/nova/scheduler/__init__.py b/nova/scheduler/__init__.py index 8359a7ae..25078f01 100644 --- a/nova/scheduler/__init__.py +++ b/nova/scheduler/__init__.py @@ -21,5 +21,7 @@ .. automodule:: nova.scheduler :platform: Unix :synopsis: Module that picks a compute node to run a VM instance. +.. moduleauthor:: Sandy Walsh +.. moduleauthor:: Ed Leafe .. moduleauthor:: Chris Behrens """ diff --git a/nova/scheduler/abstract_scheduler.py b/nova/scheduler/abstract_scheduler.py index eb924732..77db6777 100644 --- a/nova/scheduler/abstract_scheduler.py +++ b/nova/scheduler/abstract_scheduler.py @@ -14,10 +14,10 @@ # under the License. """ -The AbsractScheduler is a base class Scheduler for creating instances -across zones. There are two expansion points to this class for: -1. Assigning Weights to hosts for requested instances -2. Filtering Hosts based on required instance capabilities +The AbsractScheduler is an abstract class Scheduler for creating instances +locally or across zones. Two methods should be overridden in order to +customize the behavior: filter_hosts() and weigh_hosts(). The default +behavior is to simply select all hosts and weight them the same. """ import operator @@ -45,20 +45,19 @@ LOG = logging.getLogger('nova.scheduler.abstract_scheduler') class InvalidBlob(exception.NovaException): message = _("Ill-formed or incorrectly routed 'blob' data sent " - "to instance create request.") + "to instance create request.") class AbstractScheduler(driver.Scheduler): """Base class for creating Schedulers that can work across any nova deployment, from simple designs to multiply-nested zones. """ - def _call_zone_method(self, context, method, specs, zones): """Call novaclient zone method. Broken out for testing.""" return api.call_zone_method(context, method, specs=specs, zones=zones) def _provision_resource_locally(self, context, build_plan_item, - request_spec, kwargs): + request_spec, kwargs): """Create the requested resource in this Zone.""" host = build_plan_item['hostname'] base_options = request_spec['instance_properties'] @@ -68,21 +67,21 @@ class AbstractScheduler(driver.Scheduler): # support at some point? Also, OS API has no concept of security # groups. instance = compute_api.API().create_db_entry_for_new_instance(context, - image, base_options, None, []) + image, base_options, None, []) instance_id = instance['id'] kwargs['instance_id'] = instance_id - rpc.cast(context, - db.queue_get_for(context, "compute", host), - {"method": "run_instance", - "args": kwargs}) + queue = db.queue_get_for(context, "compute", host) + params = {"method": "run_instance", "args": kwargs} + rpc.cast(context, queue, params) LOG.debug(_("Provisioning locally via compute node %(host)s") - % locals()) + % locals()) def _decrypt_blob(self, blob): """Returns the decrypted blob or None if invalid. Broken out - for testing.""" + for testing. + """ decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key) try: json_entry = decryptor(blob) @@ -92,15 +91,15 @@ class AbstractScheduler(driver.Scheduler): return None def _ask_child_zone_to_create_instance(self, context, zone_info, - request_spec, kwargs): + request_spec, kwargs): """Once we have determined that the request should go to one of our children, we need to fabricate a new POST /servers/ call with the same parameters that were passed into us. Note that we have to reverse engineer from our args to get back the image, flavor, ipgroup, etc. since the original call could have - come in from EC2 (which doesn't use these things).""" - + come in from EC2 (which doesn't use these things). + """ instance_type = request_spec['instance_type'] instance_properties = request_spec['instance_properties'] @@ -109,30 +108,26 @@ class AbstractScheduler(driver.Scheduler): meta = instance_properties['metadata'] flavor_id = instance_type['flavorid'] reservation_id = instance_properties['reservation_id'] - files = kwargs['injected_files'] ipgroup = None # Not supported in OS API ... yet - child_zone = zone_info['child_zone'] child_blob = zone_info['child_blob'] zone = db.zone_get(context, child_zone) url = zone.api_url LOG.debug(_("Forwarding instance create call to child zone %(url)s" - ". ReservationID=%(reservation_id)s") - % locals()) + ". ReservationID=%(reservation_id)s") % locals()) nova = None try: nova = novaclient.Client(zone.username, zone.password, None, url) nova.authenticate() except novaclient_exceptions.BadRequest, e: raise exception.NotAuthorized(_("Bad credentials attempting " - "to talk to zone at %(url)s.") % locals()) - + "to talk to zone at %(url)s.") % locals()) nova.servers.create(name, image_ref, flavor_id, ipgroup, meta, files, - child_blob, reservation_id=reservation_id) + child_blob, reservation_id=reservation_id) def _provision_resource_from_blob(self, context, build_plan_item, - instance_id, request_spec, kwargs): + instance_id, request_spec, kwargs): """Create the requested resource locally or in a child zone based on what is stored in the zone blob info. @@ -145,8 +140,8 @@ class AbstractScheduler(driver.Scheduler): means we gathered the info from one of our children. It's possible that, when we decrypt the 'blob' field, it contains "child_blob" data. In which case we forward the - request.""" - + request. + """ host_info = None if "blob" in build_plan_item: # Request was passed in from above. Is it for us? @@ -161,21 +156,20 @@ class AbstractScheduler(driver.Scheduler): # Valid data ... is it for us? if 'child_zone' in host_info and 'child_blob' in host_info: self._ask_child_zone_to_create_instance(context, host_info, - request_spec, kwargs) + request_spec, kwargs) else: self._provision_resource_locally(context, host_info, request_spec, - kwargs) + kwargs) def _provision_resource(self, context, build_plan_item, instance_id, - request_spec, kwargs): + request_spec, kwargs): """Create the requested resource in this Zone or a child zone.""" if "hostname" in build_plan_item: self._provision_resource_locally(context, build_plan_item, - request_spec, kwargs) + request_spec, kwargs) return - self._provision_resource_from_blob(context, build_plan_item, - instance_id, request_spec, kwargs) + instance_id, request_spec, kwargs) def _adjust_child_weights(self, child_results, zones): """Apply the Scale and Offset values from the Zone definition @@ -185,13 +179,11 @@ class AbstractScheduler(driver.Scheduler): for zone_id, result in child_results: if not result: continue - assert isinstance(zone_id, int) for zone_rec in zones: if zone_rec['id'] != zone_id: continue - for item in result: try: offset = zone_rec['weight_offset'] @@ -202,10 +194,10 @@ class AbstractScheduler(driver.Scheduler): item['raw_weight'] = raw_weight except KeyError: LOG.exception(_("Bad child zone scaling values " - "for Zone: %(zone_id)s") % locals()) + "for Zone: %(zone_id)s") % locals()) def schedule_run_instance(self, context, instance_id, request_spec, - *args, **kwargs): + *args, **kwargs): """This method is called from nova.compute.api to provision an instance. However we need to look at the parameters being passed in to see if this is a request to: @@ -214,13 +206,11 @@ class AbstractScheduler(driver.Scheduler): to simply create the instance (either in this zone or a child zone). """ - # TODO(sandy): We'll have to look for richer specs at some point. - blob = request_spec.get('blob') if blob: self._provision_resource(context, request_spec, instance_id, - request_spec, kwargs) + request_spec, kwargs) return None num_instances = request_spec.get('num_instances', 1) @@ -235,10 +225,9 @@ class AbstractScheduler(driver.Scheduler): for num in xrange(num_instances): if not build_plan: break - build_plan_item = build_plan.pop(0) self._provision_resource(context, build_plan_item, instance_id, - request_spec, kwargs) + request_spec, kwargs) # Returning None short-circuits the routing to Compute (since # we've already done it here) @@ -251,58 +240,44 @@ class AbstractScheduler(driver.Scheduler): anything about the children. """ return self._schedule(context, "compute", request_spec, - *args, **kwargs) + *args, **kwargs) - # TODO(sandy): We're only focused on compute instances right now, - # so we don't implement the default "schedule()" method required - # of Schedulers. def schedule(self, context, topic, request_spec, *args, **kwargs): """The schedule() contract requires we return the one best-suited host for this request. """ - raise driver.NoValidHost(_('No hosts were available')) + # TODO(sandy): We're only focused on compute instances right now, + # so we don't implement the default "schedule()" method required + # of Schedulers. + msg = _("No host selection for %s defined." % topic) + raise driver.NoValidHost(msg) def _schedule(self, context, topic, request_spec, *args, **kwargs): """Returns a list of hosts that meet the required specs, ordered by their fitness. """ - if topic != "compute": - raise NotImplementedError(_("Scheduler only understands" - " Compute nodes (for now)")) + msg = _("Scheduler only understands Compute nodes (for now)") + raise NotImplementedError(msg) - num_instances = request_spec.get('num_instances', 1) - instance_type = request_spec['instance_type'] + # Get all available hosts. + all_hosts = self.zone_manager.service_states.iteritems() + unfiltered_hosts = [(host, services[topic]) + for host, services in all_hosts + if topic in services] - weighted = [] - host_list = None + # Filter local hosts based on requirements ... + filtered_hosts = self.filter_hosts(topic, request_spec, + unfiltered_hosts) + if not filtered_hosts: + LOG.warn(_("No hosts available")) + return [] - for i in xrange(num_instances): - # Filter local hosts based on requirements ... - # - # The first pass through here will pass 'None' as the - # host_list.. which tells the filter to build the full - # list of hosts. - # On a 2nd pass, the filter can modify the host_list with - # any updates it needs to make based on resources that - # may have been consumed from a previous build.. - host_list = self.filter_hosts(topic, request_spec, host_list) - if not host_list: - LOG.warn(_("Filter returned no hosts after processing " - "%(i)d of %(num_instances)d instances") % locals()) - break - - # then weigh the selected hosts. - # weighted = [{weight=weight, hostname=hostname, - # capabilities=capabs}, ...] - weights = self.weigh_hosts(topic, request_spec, host_list) - weights.sort(key=operator.itemgetter('weight')) - best_weight = weights[0] - weighted.append(best_weight) - self.consume_resources(topic, best_weight['capabilities'], - instance_type) - - # Next, tack on the best weights from the child zones ... + # weigh the selected hosts. + # weighted_hosts = [{weight=weight, hostname=hostname, + # capabilities=capabs}, ...] + weighted_hosts = self.weigh_hosts(topic, request_spec, filtered_hosts) + # Next, tack on the host weights from the child zones json_spec = json.dumps(request_spec) all_zones = db.zone_get_all(context) child_results = self._call_zone_method(context, "select", @@ -314,90 +289,32 @@ class AbstractScheduler(driver.Scheduler): # it later if needed. This implicitly builds a zone # path structure. host_dict = {"weight": weighting["weight"], - "child_zone": child_zone, - "child_blob": weighting["blob"]} - weighted.append(host_dict) + "child_zone": child_zone, + "child_blob": weighting["blob"]} + weighted_hosts.append(host_dict) + weighted_hosts.sort(key=operator.itemgetter('weight')) + return weighted_hosts - weighted.sort(key=operator.itemgetter('weight')) - return weighted + def filter_hosts(self, topic, request_spec, host_list): + """Filter the full host list returned from the ZoneManager. By default, + this method only applies the basic_ram_filter(), meaning all hosts + with at least enough RAM for the requested instance are returned. - def compute_filter(self, hostname, capabilities, request_spec): - """Return whether or not we can schedule to this compute node. - Derived classes should override this and return True if the host - is acceptable for scheduling. + Override in subclasses to provide greater selectivity. """ - instance_type = request_spec['instance_type'] - requested_mem = instance_type['memory_mb'] * 1024 * 1024 - return capabilities['host_memory_free'] >= requested_mem + def basic_ram_filter(hostname, capabilities, request_spec): + """Only return hosts with sufficient available RAM.""" + instance_type = request_spec['instance_type'] + requested_mem = instance_type['memory_mb'] * 1024 * 1024 + return capabilities['host_memory_free'] >= requested_mem - def hold_filter_hosts(self, topic, request_spec, hosts=None): - """Filter the full host list (from the ZoneManager)""" - # NOTE(dabo): The logic used by the current _schedule() method - # is incorrect. Since this task is just to refactor the classes, - # I'm not fixing the logic now - that will be the next task. - # So for now this method is just renamed; afterwards this will - # become the filter_hosts() method, and the one below will - # be removed. - filter_name = request_spec.get('filter', None) - # Make sure that the requested filter is legitimate. - selected_filter = host_filter.choose_host_filter(filter_name) - - # TODO(sandy): We're only using InstanceType-based specs - # currently. Later we'll need to snoop for more detailed - # host filter requests. - instance_type = request_spec['instance_type'] - name, query = selected_filter.instance_type_to_filter(instance_type) - return selected_filter.filter_hosts(self.zone_manager, query) - - def filter_hosts(self, topic, request_spec, host_list=None): - """Return a list of hosts which are acceptable for scheduling. - Return value should be a list of (hostname, capability_dict)s. - Derived classes may override this, but may find the - '_filter' function more appropriate. - """ - def _default_filter(self, hostname, capabilities, request_spec): - """Default filter function if there's no _filter""" - # NOTE(sirp): The default logic is the equivalent to - # AllHostsFilter - return True - - filter_func = getattr(self, '%s_filter' % topic, _default_filter) - - if host_list is None: - first_run = True - host_list = self.zone_manager.service_states.iteritems() - else: - first_run = False - - filtered_hosts = [] - for host, services in host_list: - if first_run: - if topic not in services: - continue - services = services[topic] - if filter_func(host, services, request_spec): - filtered_hosts.append((host, services)) - return filtered_hosts + return [(host, services) for host, services in host_list + if basic_ram_filter(host, services, request_spec)] def weigh_hosts(self, topic, request_spec, hosts): - """Derived classes may override this to provide more sophisticated - scheduling objectives + """This version assigns a weight of 1 to all hosts, making selection + of any host basically a random event. Override this method in your + subclass to add logic to prefer one potential host over another. """ - # NOTE(sirp): The default logic is the same as the NoopCostFunction return [dict(weight=1, hostname=hostname, capabilities=capabilities) for hostname, capabilities in hosts] - - def compute_consume(self, capabilities, instance_type): - """Consume compute resources for selected host""" - - requested_mem = max(instance_type['memory_mb'], 0) * 1024 * 1024 - capabilities['host_memory_free'] -= requested_mem - - def consume_resources(self, topic, capabilities, instance_type): - """Consume resources for a specific host. 'host' is a tuple - of the hostname and the services""" - - consume_func = getattr(self, '%s_consume' % topic, None) - if not consume_func: - return - consume_func(capabilities, instance_type) diff --git a/nova/scheduler/base_scheduler.py b/nova/scheduler/base_scheduler.py new file mode 100644 index 00000000..35e5af03 --- /dev/null +++ b/nova/scheduler/base_scheduler.py @@ -0,0 +1,59 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +The BaseScheduler is the base class Scheduler for creating instances +across zones. There are two expansion points to this class for: +1. Assigning Weights to hosts for requested instances +2. Filtering Hosts based on required instance capabilities +""" + +from nova import flags +from nova import log as logging + +from nova.scheduler import abstract_scheduler +from nova.scheduler import host_filter + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.scheduler.base_scheduler') + + +class BaseScheduler(abstract_scheduler.AbstractScheduler): + """Base class for creating Schedulers that can work across any nova + deployment, from simple designs to multiply-nested zones. + """ + def filter_hosts(self, topic, request_spec, hosts=None): + """Filter the full host list (from the ZoneManager)""" + filter_name = request_spec.get('filter', None) + # Make sure that the requested filter is legitimate. + selected_filter = host_filter.choose_host_filter(filter_name) + + # TODO(sandy): We're only using InstanceType-based specs + # currently. Later we'll need to snoop for more detailed + # host filter requests. + instance_type = request_spec.get("instance_type", None) + if instance_type is None: + # No way to select; return the specified hosts + return hosts or [] + name, query = selected_filter.instance_type_to_filter(instance_type) + return selected_filter.filter_hosts(self.zone_manager, query) + + def weigh_hosts(self, topic, request_spec, hosts): + """Derived classes may override this to provide more sophisticated + scheduling objectives + """ + # NOTE(sirp): The default logic is the same as the NoopCostFunction + return [dict(weight=1, hostname=hostname, capabilities=capabilities) + for hostname, capabilities in hosts] diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 45a8f40d..4bc5158c 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -20,283 +20,33 @@ either incompatible or insufficient to accept a newly-requested instance are removed by Host Filter classes from consideration. Those that pass the filter are then passed on for weighting or other process for ordering. -Three filters are included: AllHosts, Flavor & JSON. AllHosts just -returns the full, unfiltered list of hosts. Flavor is a hard coded -matching mechanism based on flavor criteria and JSON is an ad-hoc -filter grammar. - -Why JSON? The requests for instances may come in through the -REST interface from a user or a parent Zone. -Currently Flavors and/or InstanceTypes are used for -specifing the type of instance desired. Specific Nova users have -noted a need for a more expressive way of specifying instances. -Since we don't want to get into building full DSL this is a simple -form as an example of how this could be done. In reality, most -consumers will use the more rigid filters such as FlavorFilter. +Filters are in the 'filters' directory that is off the 'scheduler' +directory of nova. Additional filters can be created and added to that +directory; be sure to add them to the filters/__init__.py file so that +they are part of the nova.schedulers.filters namespace. """ -import json +import types from nova import exception from nova import flags -from nova import log as logging -from nova import utils +import nova.scheduler -LOG = logging.getLogger('nova.scheduler.host_filter') FLAGS = flags.FLAGS -flags.DEFINE_string('default_host_filter', - 'nova.scheduler.host_filter.AllHostsFilter', - 'Which filter to use for filtering hosts.') -class HostFilter(object): - """Base class for host filters.""" +def _get_filters(): + # Imported here to avoid circular imports + from nova.scheduler import filters - def instance_type_to_filter(self, instance_type): - """Convert instance_type into a filter for most common use-case.""" - raise NotImplementedError() + def get_itm(nm): + return getattr(filters, nm) - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that fulfill the filter.""" - raise NotImplementedError() - - def _full_name(self): - """module.classname of the filter.""" - return "%s.%s" % (self.__module__, self.__class__.__name__) - - -class AllHostsFilter(HostFilter): - """ NOP host filter. Returns all hosts in ZoneManager. - This essentially does what the old Scheduler+Chance used - to give us. - """ - - def instance_type_to_filter(self, instance_type): - """Return anything to prevent base-class from raising - exception.""" - return (self._full_name(), instance_type) - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts from ZoneManager list.""" - return [(host, services) - for host, services in zone_manager.service_states.iteritems()] - - -class InstanceTypeFilter(HostFilter): - """HostFilter hard-coded to work with InstanceType records.""" - - def instance_type_to_filter(self, instance_type): - """Use instance_type to filter hosts.""" - return (self._full_name(), instance_type) - - def _satisfies_extra_specs(self, capabilities, instance_type): - """Check that the capabilities provided by the compute service - satisfy the extra specs associated with the instance type""" - - if 'extra_specs' not in instance_type: - return True - - # Note(lorinh): For now, we are just checking exact matching on the - # values. Later on, we want to handle numerical - # values so we can represent things like number of GPU cards - - try: - for key, value in instance_type['extra_specs'].iteritems(): - if capabilities[key] != value: - return False - except KeyError: - return False - - return True - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that can create instance_type.""" - instance_type = query - selected_hosts = [] - for host, services in zone_manager.service_states.iteritems(): - capabilities = services.get('compute', {}) - host_ram_mb = capabilities['host_memory_free'] - disk_bytes = capabilities['disk_available'] - spec_ram = instance_type['memory_mb'] - spec_disk = instance_type['local_gb'] - extra_specs = instance_type['extra_specs'] - - if ((host_ram_mb >= spec_ram) and (disk_bytes >= spec_disk) and - self._satisfies_extra_specs(capabilities, instance_type)): - selected_hosts.append((host, capabilities)) - return selected_hosts - -#host entries (currently) are like: -# {'host_name-description': 'Default install of XenServer', -# 'host_hostname': 'xs-mini', -# 'host_memory_total': 8244539392, -# 'host_memory_overhead': 184225792, -# 'host_memory_free': 3868327936, -# 'host_memory_free_computed': 3840843776, -# 'host_other_config': {}, -# 'host_ip_address': '192.168.1.109', -# 'host_cpu_info': {}, -# 'disk_available': 32954957824, -# 'disk_total': 50394562560, -# 'disk_used': 17439604736, -# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', -# 'host_name_label': 'xs-mini'} - -# instance_type table has: -#name = Column(String(255), unique=True) -#memory_mb = Column(Integer) -#vcpus = Column(Integer) -#local_gb = Column(Integer) -#flavorid = Column(Integer, unique=True) -#swap = Column(Integer, nullable=False, default=0) -#rxtx_quota = Column(Integer, nullable=False, default=0) -#rxtx_cap = Column(Integer, nullable=False, default=0) - - -class JsonFilter(HostFilter): - """Host Filter to allow simple JSON-based grammar for - selecting hosts. - """ - - def _equals(self, args): - """First term is == all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs != rhs: - return False - return True - - def _less_than(self, args): - """First term is < all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs >= rhs: - return False - return True - - def _greater_than(self, args): - """First term is > all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs <= rhs: - return False - return True - - def _in(self, args): - """First term is in set of remaining terms""" - if len(args) < 2: - return False - return args[0] in args[1:] - - def _less_than_equal(self, args): - """First term is <= all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs > rhs: - return False - return True - - def _greater_than_equal(self, args): - """First term is >= all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs < rhs: - return False - return True - - def _not(self, args): - """Flip each of the arguments.""" - if len(args) == 0: - return False - return [not arg for arg in args] - - def _or(self, args): - """True if any arg is True.""" - return True in args - - def _and(self, args): - """True if all args are True.""" - return False not in args - - commands = { - '=': _equals, - '<': _less_than, - '>': _greater_than, - 'in': _in, - '<=': _less_than_equal, - '>=': _greater_than_equal, - 'not': _not, - 'or': _or, - 'and': _and, - } - - def instance_type_to_filter(self, instance_type): - """Convert instance_type into JSON filter object.""" - required_ram = instance_type['memory_mb'] - required_disk = instance_type['local_gb'] - query = ['and', - ['>=', '$compute.host_memory_free', required_ram], - ['>=', '$compute.disk_available', required_disk]] - return (self._full_name(), json.dumps(query)) - - def _parse_string(self, string, host, services): - """Strings prefixed with $ are capability lookups in the - form '$service.capability[.subcap*]' - """ - if not string: - return None - if string[0] != '$': - return string - - path = string[1:].split('.') - for item in path: - services = services.get(item, None) - if not services: - return None - return services - - def _process_filter(self, zone_manager, query, host, services): - """Recursively parse the query structure.""" - if len(query) == 0: - return True - cmd = query[0] - method = self.commands[cmd] # Let exception fly. - cooked_args = [] - for arg in query[1:]: - if isinstance(arg, list): - arg = self._process_filter(zone_manager, arg, host, services) - elif isinstance(arg, basestring): - arg = self._parse_string(arg, host, services) - if arg != None: - cooked_args.append(arg) - result = method(self, cooked_args) - return result - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that can fulfill filter.""" - expanded = json.loads(query) - hosts = [] - for host, services in zone_manager.service_states.iteritems(): - r = self._process_filter(zone_manager, expanded, host, services) - if isinstance(r, list): - r = True in r - if r: - hosts.append((host, services)) - return hosts - - -FILTERS = [AllHostsFilter, InstanceTypeFilter, JsonFilter] + return [get_itm(itm) for itm in dir(filters) + if (type(get_itm(itm)) is types.TypeType) + and issubclass(get_itm(itm), filters.AbstractHostFilter) + and get_itm(itm) is not filters.AbstractHostFilter] def choose_host_filter(filter_name=None): @@ -307,8 +57,9 @@ def choose_host_filter(filter_name=None): """ if not filter_name: filter_name = FLAGS.default_host_filter - for filter_class in FILTERS: + for filter_class in _get_filters(): host_match = "%s.%s" % (filter_class.__module__, filter_class.__name__) - if host_match == filter_name: + if (host_match.startswith("nova.scheduler.filters") and + (host_match.split(".")[-1] == filter_name)): return filter_class() raise exception.SchedulerHostFilterNotFound(filter_name=filter_name) diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py index a58b1128..903d786c 100644 --- a/nova/scheduler/least_cost.py +++ b/nova/scheduler/least_cost.py @@ -22,14 +22,12 @@ The cost-function and weights are tabulated, and the host with the least cost is then selected for provisioning. """ -# TODO(dabo): This class will be removed in the next merge prop; it remains now -# because much of the code will be refactored into different classes. import collections from nova import flags from nova import log as logging -from nova.scheduler import abstract_scheduler +from nova.scheduler import base_scheduler from nova import utils from nova import exception @@ -37,14 +35,16 @@ LOG = logging.getLogger('nova.scheduler.least_cost') FLAGS = flags.FLAGS flags.DEFINE_list('least_cost_scheduler_cost_functions', - ['nova.scheduler.least_cost.noop_cost_fn'], - 'Which cost functions the LeastCostScheduler should use.') + ['nova.scheduler.least_cost.noop_cost_fn'], + 'Which cost functions the LeastCostScheduler should use.') # TODO(sirp): Once we have enough of these rules, we can break them out into a # cost_functions.py file (perhaps in a least_cost_scheduler directory) flags.DEFINE_integer('noop_cost_fn_weight', 1, - 'How much weight to give the noop cost function') + 'How much weight to give the noop cost function') +flags.DEFINE_integer('compute_fill_first_cost_fn_weight', 1, + 'How much weight to give the fill-first cost function') def noop_cost_fn(host): @@ -52,87 +52,20 @@ def noop_cost_fn(host): return 1 -flags.DEFINE_integer('compute_fill_first_cost_fn_weight', 1, - 'How much weight to give the fill-first cost function') - - def compute_fill_first_cost_fn(host): """Prefer hosts that have less ram available, filter_hosts will exclude - hosts that don't have enough ram""" - hostname, caps = host - free_mem = caps['host_memory_free'] + hosts that don't have enough ram. + """ + hostname, service = host + caps = service.get("compute", {}) + free_mem = caps.get("host_memory_free", 0) return free_mem -class LeastCostScheduler(abstract_scheduler.AbstractScheduler): - def __init__(self, *args, **kwargs): - self.cost_fns_cache = {} - super(LeastCostScheduler, self).__init__(*args, **kwargs) - - def get_cost_fns(self, topic): - """Returns a list of tuples containing weights and cost functions to - use for weighing hosts - """ - - if topic in self.cost_fns_cache: - return self.cost_fns_cache[topic] - - cost_fns = [] - for cost_fn_str in FLAGS.least_cost_scheduler_cost_functions: - if '.' in cost_fn_str: - short_name = cost_fn_str.split('.')[-1] - else: - short_name = cost_fn_str - cost_fn_str = "%s.%s.%s" % ( - __name__, self.__class__.__name__, short_name) - - if not (short_name.startswith('%s_' % topic) or - short_name.startswith('noop')): - continue - - try: - # NOTE(sirp): import_class is somewhat misnamed since it can - # any callable from a module - cost_fn = utils.import_class(cost_fn_str) - except exception.ClassNotFound: - raise exception.SchedulerCostFunctionNotFound( - cost_fn_str=cost_fn_str) - - try: - flag_name = "%s_weight" % cost_fn.__name__ - weight = getattr(FLAGS, flag_name) - except AttributeError: - raise exception.SchedulerWeightFlagNotFound( - flag_name=flag_name) - - cost_fns.append((weight, cost_fn)) - - self.cost_fns_cache[topic] = cost_fns - return cost_fns - - def weigh_hosts(self, topic, request_spec, hosts): - """Returns a list of dictionaries of form: - [ {weight: weight, hostname: hostname, capabilities: capabs} ] - """ - - cost_fns = self.get_cost_fns(topic) - costs = weighted_sum(domain=hosts, weighted_fns=cost_fns) - - weighted = [] - weight_log = [] - for cost, (hostname, caps) in zip(costs, hosts): - weight_log.append("%s: %s" % (hostname, "%.2f" % cost)) - weight_dict = dict(weight=cost, hostname=hostname, - capabilities=caps) - weighted.append(weight_dict) - - LOG.debug(_("Weighted Costs => %s") % weight_log) - return weighted - - def normalize_list(L): """Normalize an array of numbers such that each element satisfies: - 0 <= e <= 1""" + 0 <= e <= 1 + """ if not L: return L max_ = max(L) @@ -160,12 +93,10 @@ def weighted_sum(domain, weighted_fns, normalize=True): score_table = collections.defaultdict(list) for weight, fn in weighted_fns: scores = [fn(elem) for elem in domain] - if normalize: norm_scores = normalize_list(scores) else: norm_scores = scores - for idx, score in enumerate(norm_scores): weighted_score = score * weight score_table[idx].append(weighted_score) @@ -175,5 +106,66 @@ def weighted_sum(domain, weighted_fns, normalize=True): for idx in sorted(score_table): elem_score = sum(score_table[idx]) domain_scores.append(elem_score) - return domain_scores + + +class LeastCostScheduler(base_scheduler.BaseScheduler): + def __init__(self, *args, **kwargs): + self.cost_fns_cache = {} + super(LeastCostScheduler, self).__init__(*args, **kwargs) + + def get_cost_fns(self, topic): + """Returns a list of tuples containing weights and cost functions to + use for weighing hosts + """ + if topic in self.cost_fns_cache: + return self.cost_fns_cache[topic] + cost_fns = [] + for cost_fn_str in FLAGS.least_cost_scheduler_cost_functions: + if '.' in cost_fn_str: + short_name = cost_fn_str.split('.')[-1] + else: + short_name = cost_fn_str + cost_fn_str = "%s.%s.%s" % ( + __name__, self.__class__.__name__, short_name) + if not (short_name.startswith('%s_' % topic) or + short_name.startswith('noop')): + continue + + try: + # NOTE(sirp): import_class is somewhat misnamed since it can + # any callable from a module + cost_fn = utils.import_class(cost_fn_str) + except exception.ClassNotFound: + raise exception.SchedulerCostFunctionNotFound( + cost_fn_str=cost_fn_str) + + try: + flag_name = "%s_weight" % cost_fn.__name__ + weight = getattr(FLAGS, flag_name) + except AttributeError: + raise exception.SchedulerWeightFlagNotFound( + flag_name=flag_name) + cost_fns.append((weight, cost_fn)) + + self.cost_fns_cache[topic] = cost_fns + return cost_fns + + def weigh_hosts(self, topic, request_spec, hosts): + """Returns a list of dictionaries of form: + [ {weight: weight, hostname: hostname, capabilities: capabs} ] + """ + cost_fns = self.get_cost_fns(topic) + costs = weighted_sum(domain=hosts, weighted_fns=cost_fns) + + weighted = [] + weight_log = [] + for cost, (hostname, service) in zip(costs, hosts): + caps = service[topic] + weight_log.append("%s: %s" % (hostname, "%.2f" % cost)) + weight_dict = dict(weight=cost, hostname=hostname, + capabilities=caps) + weighted.append(weight_dict) + + LOG.debug(_("Weighted Costs => %s") % weight_log) + return weighted diff --git a/nova/tests/scheduler/test_abstract_scheduler.py b/nova/tests/scheduler/test_abstract_scheduler.py index f4f5cc23..aa97e234 100644 --- a/nova/tests/scheduler/test_abstract_scheduler.py +++ b/nova/tests/scheduler/test_abstract_scheduler.py @@ -77,6 +77,9 @@ class FakeZoneManager(zone_manager.ZoneManager): 'host3': { 'compute': {'host_memory_free': 3221225472}, }, + 'host4': { + 'compute': {'host_memory_free': 999999999}, + }, } diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py index 7e664d3f..17431fc7 100644 --- a/nova/tests/scheduler/test_host_filter.py +++ b/nova/tests/scheduler/test_host_filter.py @@ -21,6 +21,7 @@ import json from nova import exception from nova import test from nova.scheduler import host_filter +from nova.scheduler import filters class FakeZoneManager: @@ -55,7 +56,7 @@ class HostFilterTestCase(test.TestCase): def setUp(self): super(HostFilterTestCase, self).setUp() - default_host_filter = 'nova.scheduler.host_filter.AllHostsFilter' + default_host_filter = 'AllHostsFilter' self.flags(default_host_filter=default_host_filter) self.instance_type = dict(name='tiny', memory_mb=50, @@ -98,13 +99,10 @@ class HostFilterTestCase(test.TestCase): def test_choose_filter(self): # Test default filter ... hf = host_filter.choose_host_filter() - self.assertEquals(hf._full_name(), - 'nova.scheduler.host_filter.AllHostsFilter') + self.assertEquals(hf._full_name().split(".")[-1], 'AllHostsFilter') # Test valid filter ... - hf = host_filter.choose_host_filter( - 'nova.scheduler.host_filter.InstanceTypeFilter') - self.assertEquals(hf._full_name(), - 'nova.scheduler.host_filter.InstanceTypeFilter') + hf = host_filter.choose_host_filter('InstanceTypeFilter') + self.assertEquals(hf._full_name().split(".")[-1], 'InstanceTypeFilter') # Test invalid filter ... try: host_filter.choose_host_filter('does not exist') @@ -113,7 +111,7 @@ class HostFilterTestCase(test.TestCase): pass def test_all_host_filter(self): - hf = host_filter.AllHostsFilter() + hf = filters.AllHostsFilter() cooked = hf.instance_type_to_filter(self.instance_type) hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(10, len(hosts)) @@ -121,11 +119,10 @@ class HostFilterTestCase(test.TestCase): self.assertTrue(host.startswith('host')) def test_instance_type_filter(self): - hf = host_filter.InstanceTypeFilter() + hf = filters.InstanceTypeFilter() # filter all hosts that can support 50 ram and 500 disk name, cooked = hf.instance_type_to_filter(self.instance_type) - self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', - name) + self.assertEquals(name.split(".")[-1], 'InstanceTypeFilter') hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -134,21 +131,20 @@ class HostFilterTestCase(test.TestCase): self.assertEquals('host10', just_hosts[5]) def test_instance_type_filter_extra_specs(self): - hf = host_filter.InstanceTypeFilter() + hf = filters.InstanceTypeFilter() # filter all hosts that can support 50 ram and 500 disk name, cooked = hf.instance_type_to_filter(self.gpu_instance_type) - self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', - name) + self.assertEquals(name.split(".")[-1], 'InstanceTypeFilter') hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(1, len(hosts)) just_hosts = [host for host, caps in hosts] self.assertEquals('host07', just_hosts[0]) def test_json_filter(self): - hf = host_filter.JsonFilter() + hf = filters.JsonFilter() # filter all hosts that can support 50 ram and 500 disk name, cooked = hf.instance_type_to_filter(self.instance_type) - self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) + self.assertEquals(name.split(".")[-1], 'JsonFilter') hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -192,7 +188,6 @@ class HostFilterTestCase(test.TestCase): raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) hosts = hf.filter_hosts(self.zone_manager, cooked) - self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() diff --git a/nova/tests/scheduler/test_least_cost_scheduler.py b/nova/tests/scheduler/test_least_cost_scheduler.py index de7581d0..af58de52 100644 --- a/nova/tests/scheduler/test_least_cost_scheduler.py +++ b/nova/tests/scheduler/test_least_cost_scheduler.py @@ -15,6 +15,7 @@ """ Tests For Least Cost Scheduler """ +import copy from nova import test from nova.scheduler import least_cost @@ -81,7 +82,7 @@ class LeastCostSchedulerTestCase(test.TestCase): super(LeastCostSchedulerTestCase, self).tearDown() def assertWeights(self, expected, num, request_spec, hosts): - weighted = self.sched.weigh_hosts(num, request_spec, hosts) + weighted = self.sched.weigh_hosts("compute", request_spec, hosts) self.assertDictListMatch(weighted, expected, approx_equal=True) def test_no_hosts(self): @@ -122,19 +123,24 @@ class LeastCostSchedulerTestCase(test.TestCase): self.flags(least_cost_scheduler_cost_functions=[ 'nova.scheduler.least_cost.compute_fill_first_cost_fn'], compute_fill_first_cost_fn_weight=1) - num = 1 instance_type = {'memory_mb': 1024} request_spec = {'instance_type': instance_type} - hosts = self.sched.filter_hosts('compute', request_spec, None) + svc_states = self.sched.zone_manager.service_states.iteritems() + all_hosts = [(host, services["compute"]) + for host, services in svc_states + if "compute" in services] + hosts = self.sched.filter_hosts('compute', request_spec, all_hosts) expected = [] - for idx, (hostname, caps) in enumerate(hosts): + for idx, (hostname, services) in enumerate(hosts): + caps = copy.deepcopy(services["compute"]) # Costs are normalized so over 10 hosts, each host with increasing # free ram will cost 1/N more. Since the lowest cost host has some # free ram, we add in the 1/N for the base_cost weight = 0.1 + (0.1 * idx) - weight_dict = dict(weight=weight, hostname=hostname) - expected.append(weight_dict) + wtd_dict = dict(hostname=hostname, weight=weight, + capabilities=caps) + expected.append(wtd_dict) self.assertWeights(expected, num, request_spec, hosts) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index b2afc53c..0793784f 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -487,6 +487,17 @@ class CloudTestCase(test.TestCase): db.service_destroy(self.context, comp1['id']) db.service_destroy(self.context, comp2['id']) + def test_describe_instances_deleted(self): + args1 = {'reservation_id': 'a', 'image_ref': 1, 'host': 'host1'} + inst1 = db.instance_create(self.context, args1) + args2 = {'reservation_id': 'b', 'image_ref': 1, 'host': 'host1'} + inst2 = db.instance_create(self.context, args2) + db.instance_destroy(self.context, inst1.id) + result = self.cloud.describe_instances(self.context) + result = result['reservationSet'][0]['instancesSet'] + self.assertEqual(result[0]['instanceId'], + ec2utils.id_to_ec2_id(inst2.id)) + def _block_device_mapping_create(self, instance_id, mappings): volumes = [] for bdm in mappings: diff --git a/nova/tests/test_db_api.py b/nova/tests/test_db_api.py index 0c07cbb7..038c07f4 100644 --- a/nova/tests/test_db_api.py +++ b/nova/tests/test_db_api.py @@ -76,3 +76,20 @@ class DbApiTestCase(test.TestCase): self.assertEqual(instance['id'], result['id']) self.assertEqual(result['fixed_ips'][0]['floating_ips'][0].address, '1.2.1.2') + + def test_instance_get_all_by_filters(self): + args = {'reservation_id': 'a', 'image_ref': 1, 'host': 'host1'} + inst1 = db.instance_create(self.context, args) + inst2 = db.instance_create(self.context, args) + result = db.instance_get_all_by_filters(self.context, {}) + self.assertTrue(2, len(result)) + + def test_instance_get_all_by_filters_deleted(self): + args1 = {'reservation_id': 'a', 'image_ref': 1, 'host': 'host1'} + inst1 = db.instance_create(self.context, args1) + args2 = {'reservation_id': 'b', 'image_ref': 1, 'host': 'host1'} + inst2 = db.instance_create(self.context, args2) + db.instance_destroy(self.context, inst1.id) + result = db.instance_get_all_by_filters(self.context.elevated(), {}) + self.assertEqual(1, len(result)) + self.assertEqual(result[0].id, inst2.id) diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py deleted file mode 100644 index 3a1389a4..00000000 --- a/nova/tests/test_host_filter.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Tests For Scheduler Host Filters. -""" - -import json - -from nova import exception -from nova import test -from nova.scheduler import host_filter - - -class FakeZoneManager: - pass - - -class HostFilterTestCase(test.TestCase): - """Test case for host filters.""" - - def _host_caps(self, multiplier): - # Returns host capabilities in the following way: - # host1 = memory:free 10 (100max) - # disk:available 100 (1000max) - # hostN = memory:free 10 + 10N - # disk:available 100 + 100N - # in other words: hostN has more resources than host0 - # which means ... don't go above 10 hosts. - return {'host_name-description': 'XenServer %s' % multiplier, - 'host_hostname': 'xs-%s' % multiplier, - 'host_memory_total': 100, - 'host_memory_overhead': 10, - 'host_memory_free': 10 + multiplier * 10, - 'host_memory_free-computed': 10 + multiplier * 10, - 'host_other-config': {}, - 'host_ip_address': '192.168.1.%d' % (100 + multiplier), - 'host_cpu_info': {}, - 'disk_available': 100 + multiplier * 100, - 'disk_total': 1000, - 'disk_used': 0, - 'host_uuid': 'xxx-%d' % multiplier, - 'host_name-label': 'xs-%s' % multiplier} - - def setUp(self): - super(HostFilterTestCase, self).setUp() - default_host_filter = 'nova.scheduler.host_filter.AllHostsFilter' - self.flags(default_host_filter=default_host_filter) - self.instance_type = dict(name='tiny', - memory_mb=50, - vcpus=10, - local_gb=500, - flavorid=1, - swap=500, - rxtx_quota=30000, - rxtx_cap=200, - extra_specs={}) - - self.zone_manager = FakeZoneManager() - states = {} - for x in xrange(10): - states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} - self.zone_manager.service_states = states - - def test_choose_filter(self): - # Test default filter ... - hf = host_filter.choose_host_filter() - self.assertEquals(hf._full_name(), - 'nova.scheduler.host_filter.AllHostsFilter') - # Test valid filter ... - hf = host_filter.choose_host_filter( - 'nova.scheduler.host_filter.InstanceTypeFilter') - self.assertEquals(hf._full_name(), - 'nova.scheduler.host_filter.InstanceTypeFilter') - # Test invalid filter ... - try: - host_filter.choose_host_filter('does not exist') - self.fail("Should not find host filter.") - except exception.SchedulerHostFilterNotFound: - pass - - def test_all_host_filter(self): - hf = host_filter.AllHostsFilter() - cooked = hf.instance_type_to_filter(self.instance_type) - hosts = hf.filter_hosts(self.zone_manager, cooked) - self.assertEquals(10, len(hosts)) - for host, capabilities in hosts: - self.assertTrue(host.startswith('host')) - - def test_instance_type_filter(self): - hf = host_filter.InstanceTypeFilter() - # filter all hosts that can support 50 ram and 500 disk - name, cooked = hf.instance_type_to_filter(self.instance_type) - self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', - name) - hosts = hf.filter_hosts(self.zone_manager, cooked) - self.assertEquals(6, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - self.assertEquals('host05', just_hosts[0]) - self.assertEquals('host10', just_hosts[5]) - - def test_json_filter(self): - hf = host_filter.JsonFilter() - # filter all hosts that can support 50 ram and 500 disk - name, cooked = hf.instance_type_to_filter(self.instance_type) - self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) - hosts = hf.filter_hosts(self.zone_manager, cooked) - self.assertEquals(6, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - self.assertEquals('host05', just_hosts[0]) - self.assertEquals('host10', just_hosts[5]) - - # Try some custom queries - - raw = ['or', - ['and', - ['<', '$compute.host_memory_free', 30], - ['<', '$compute.disk_available', 300], - ], - ['and', - ['>', '$compute.host_memory_free', 70], - ['>', '$compute.disk_available', 700], - ], - ] - - cooked = json.dumps(raw) - hosts = hf.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(5, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([1, 2, 8, 9, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - raw = ['not', - ['=', '$compute.host_memory_free', 30], - ] - cooked = json.dumps(raw) - hosts = hf.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(9, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] - cooked = json.dumps(raw) - hosts = hf.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(5, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([2, 4, 6, 8, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - # Try some bogus input ... - raw = ['unknown command', ] - cooked = json.dumps(raw) - try: - hf.filter_hosts(self.zone_manager, cooked) - self.fail("Should give KeyError") - except KeyError, e: - pass - - self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps([]))) - self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps({}))) - self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps( - ['not', True, False, True, False]))) - - try: - hf.filter_hosts(self.zone_manager, json.dumps( - 'not', True, False, True, False)) - self.fail("Should give KeyError") - except KeyError, e: - pass - - self.assertFalse(hf.filter_hosts(self.zone_manager, - json.dumps(['=', '$foo', 100]))) - self.assertFalse(hf.filter_hosts(self.zone_manager, - json.dumps(['=', '$.....', 100]))) - self.assertFalse(hf.filter_hosts(self.zone_manager, - json.dumps( - ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]]))) - - self.assertFalse(hf.filter_hosts(self.zone_manager, - json.dumps(['=', {}, ['>', '$missing....foo']]))) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 688518bb..6a213b4f 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -836,6 +836,7 @@ class LibvirtConnTestCase(test.TestCase): count = (0 <= str(e.message).find('Unexpected method call')) shutil.rmtree(os.path.join(FLAGS.instances_path, instance.name)) + shutil.rmtree(os.path.join(FLAGS.instances_path, '_base')) self.assertTrue(count) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 581a4765..86fa9ee7 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -110,11 +110,14 @@ floating_ip_fields = {'id': 0, vifs = [{'id': 0, 'address': 'DE:AD:BE:EF:00:00', + 'uuid': '00000000-0000-0000-0000-0000000000000000', 'network_id': 0, 'network': FakeModel(**networks[0]), 'instance_id': 0}, {'id': 1, 'address': 'DE:AD:BE:EF:00:01', + 'uuid': '00000000-0000-0000-0000-0000000000000001', + 'network_id': 0, 'network_id': 1, 'network': FakeModel(**networks[1]), 'instance_id': 0}] @@ -165,6 +168,8 @@ class FlatNetworkTestCase(test.TestCase): 'ips': 'DONTCARE', 'label': 'test%s' % i, 'mac': 'DE:AD:BE:EF:00:0%s' % i, + 'vif_uuid': ('00000000-0000-0000-0000-000000000000000%s' % + i), 'rxtx_cap': 'DONTCARE', 'should_create_vlan': False, 'should_create_bridge': False}