Phew ... ok, this is the last dist-scheduler merge before we get into serious testing and minor tweaks. The heavy lifting is largely done.

This branch adds an OS API POST /zone/boot command which returns a reservation ID (unlike POST /servers which returns a single instance_id). 

This branch requires v2.5 of python-novaclient

Additionally GET /servers can now take an optional reservation_id parameter, which will return all the instances with that reservation ID across all zones.

nova/scheduler/api.py

@@ -111,7 +111,8 @@ def _process(func, zone):
     return func(nova, zone)
 
 
-def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs):
+def call_zone_method(context, method_name, errors_to_ignore=None,
+                     novaclient_collection_name='zones', *args, **kwargs):
     """Returns a list of (zone, call_result) objects."""
     if not isinstance(errors_to_ignore, (list, tuple)):
         # This will also handle the default None
@@ -131,18 +132,16 @@ def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs):
             #TODO (dabo) - add logic for failure counts per zone,
             # with escalation after a given number of failures.
             continue
-        zone_method = getattr(nova.zones, method)
+        novaclient_collection = getattr(nova, novaclient_collection_name)
+        collection_method = getattr(novaclient_collection, method_name)
 
         def _error_trap(*args, **kwargs):
             try:
-                return zone_method(*args, **kwargs)
+                return collection_method(*args, **kwargs)
             except Exception as e:
                 if type(e) in errors_to_ignore:
                     return None
-                # TODO (dabo) - want to be able to re-raise here.
+                raise
-                # Returning a string now; raising was causing issues.
-                # raise e
-                return "ERROR", "%s" % e
 
         res = pool.spawn(_error_trap, *args, **kwargs)
         results.append((zone, res))

nova/scheduler/manager.py

@@ -89,8 +89,8 @@ class SchedulerManager(manager.Manager):
             host = getattr(self.driver, driver_method)(elevated, *args,
                                                        **kwargs)
         except AttributeError, e:
-            LOG.exception(_("Driver Method %(driver_method)s missing: %(e)s")
+            LOG.warning(_("Driver Method %(driver_method)s missing: %(e)s."
-                                % locals())
+                            "Reverting to schedule()") % locals())
             host = self.driver.schedule(elevated, topic, *args, **kwargs)
 
         if not host:

nova/scheduler/zone_aware_scheduler.py

@@ -91,6 +91,7 @@ class ZoneAwareScheduler(driver.Scheduler):
         image_id = instance_properties['image_id']
         meta = instance_properties['metadata']
         flavor_id = instance_type['flavorid']
+        reservation_id = instance_properties['reservation_id']
 
         files = kwargs['injected_files']
         ipgroup = None  # Not supported in OS API ... yet
@@ -99,7 +100,8 @@ class ZoneAwareScheduler(driver.Scheduler):
         child_blob = zone_info['child_blob']
         zone = db.zone_get(context, child_zone)
         url = zone.api_url
-        LOG.debug(_("Forwarding instance create call to child zone %(url)s")
+        LOG.debug(_("Forwarding instance create call to child zone %(url)s"
+                    ". ReservationID=%(reservation_id)s")
                     % locals())
         nova = None
         try:
@@ -110,7 +112,7 @@ class ZoneAwareScheduler(driver.Scheduler):
                             "to talk to zone at %(url)s.") % locals())
 
         nova.servers.create(name, image_id, flavor_id, ipgroup, meta, files,
-                            child_blob)
+                            child_blob, reservation_id=reservation_id)
 
     def _provision_resource_from_blob(self, context, item, instance_id,
                                           request_spec, kwargs):

nova/tests/scheduler/test_scheduler.py

@@ -1110,10 +1110,4 @@ class CallZoneMethodTest(test.TestCase):
     def test_call_zone_method_generates_exception(self):
         context = {}
         method = 'raises_exception'
-        results = api.call_zone_method(context, method)
+        self.assertRaises(Exception, api.call_zone_method, context, method)
-
-        # FIXME(sirp): for now the _error_trap code is catching errors and
-        # converting them to a ("ERROR", "string") tuples. The code (and this
-        # test) should eventually handle real exceptions.
-        expected = [(1, ('ERROR', 'testing'))]
-        self.assertEqual(expected, results)