merged trunk again

Authors

@@ -31,6 +31,7 @@ Hisaharu Ishii <ishii.hisaharu@lab.ntt.co.jp>
 Hisaki Ohara <hisaki.ohara@intel.com>
 Ilya Alekseyev <ialekseev@griddynamics.com>
 Isaku Yamahata <yamahata@valinux.co.jp>
+Jason Cannavale <jason.cannavale@rackspace.com>
 Jason Koelker <jason@koelker.net>
 Jay Pipes <jaypipes@gmail.com>
 Jesse Andrews <anotherjesse@gmail.com>

nova/auth/ldapdriver.py

@@ -139,7 +139,7 @@ class LdapDriver(object):
         self.__cache = None
         return False
 
-    def __local_cache(key_fmt):
+    def __local_cache(key_fmt):  # pylint: disable=E0213
         """Wrap function to cache it's result in self.__cache.
         Works only with functions with one fixed argument.
         """

nova/auth/novarc.template

@@ -14,4 +14,5 @@ alias ec2-bundle-image="ec2-bundle-image --cert ${EC2_CERT} --privatekey ${EC2_P
 alias ec2-upload-bundle="ec2-upload-bundle -a ${EC2_ACCESS_KEY} -s ${EC2_SECRET_KEY} --url ${S3_URL} --ec2cert ${NOVA_CERT}"
 export NOVA_API_KEY="%(access)s"
 export NOVA_USERNAME="%(user)s"
+export NOVA_PROJECT_ID="%(project)s"
 export NOVA_URL="%(os)s"

nova/flags.py

@@ -381,3 +381,5 @@ DEFINE_string('zone_name', 'nova', 'name of this zone')
 DEFINE_list('zone_capabilities',
                 ['hypervisor=xenserver;kvm', 'os=linux;windows'],
                  'Key/Multi-value list representng capabilities of this zone')
+DEFINE_string('build_plan_encryption_key', None,
+        '128bit (hex) encryption key for scheduler build plans.')

nova/scheduler/api.py

@@ -84,7 +84,7 @@ def get_zone_capabilities(context):
 def select(context, specs=None):
     """Returns a list of hosts."""
     return _call_scheduler('select', context=context,
-            params={"specs": specs})
+            params={"request_spec": specs})
 
 
 def update_service_capabilities(context, service_name, host, capabilities):

nova/scheduler/zone_aware_scheduler.py

@@ -21,16 +21,30 @@ across zones. There are two expansion points to this class for:
 """
 
 import operator
+import json
 
+import M2Crypto
+import novaclient
+
+from nova import crypto
 from nova import db
+from nova import exception
+from nova import flags
 from nova import log as logging
 from nova import rpc
+
 from nova.scheduler import api
 from nova.scheduler import driver
 
+FLAGS = flags.FLAGS
 LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler')
 
 
+class InvalidBlob(exception.NovaException):
+    message = _("Ill-formed or incorrectly routed 'blob' data sent "
+                "to instance create request.")
+
+
 class ZoneAwareScheduler(driver.Scheduler):
     """Base class for creating Zone Aware Schedulers."""
 
@@ -38,6 +52,112 @@ class ZoneAwareScheduler(driver.Scheduler):
         """Call novaclient zone method. Broken out for testing."""
         return api.call_zone_method(context, method, specs=specs)
 
+    def _provision_resource_locally(self, context, item, instance_id, kwargs):
+        """Create the requested resource in this Zone."""
+        host = item['hostname']
+        kwargs['instance_id'] = instance_id
+        rpc.cast(context,
+                 db.queue_get_for(context, "compute", host),
+                 {"method": "run_instance",
+                  "args": kwargs})
+        LOG.debug(_("Provisioning locally via compute node %(host)s")
+                            % locals())
+
+    def _decrypt_blob(self, blob):
+        """Returns the decrypted blob or None if invalid. Broken out
+        for testing."""
+        decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key)
+        try:
+            json_entry = decryptor(blob)
+            return json.dumps(entry)
+        except M2Crypto.EVP.EVPError:
+            pass
+        return None
+
+    def _ask_child_zone_to_create_instance(self, context, zone_info,
+                                           request_spec, kwargs):
+        """Once we have determined that the request should go to one
+        of our children, we need to fabricate a new POST /servers/
+        call with the same parameters that were passed into us.
+
+        Note that we have to reverse engineer from our args to get back the
+        image, flavor, ipgroup, etc. since the original call could have
+        come in from EC2 (which doesn't use these things)."""
+
+        instance_type = request_spec['instance_type']
+        instance_properties = request_spec['instance_properties']
+
+        name = instance_properties['display_name']
+        image_id = instance_properties['image_id']
+        meta = instance_properties['metadata']
+        flavor_id = instance_type['flavorid']
+
+        files = kwargs['injected_files']
+        ipgroup = None  # Not supported in OS API ... yet
+
+        child_zone = zone_info['child_zone']
+        child_blob = zone_info['child_blob']
+        zone = db.zone_get(context, child_zone)
+        url = zone.api_url
+        LOG.debug(_("Forwarding instance create call to child zone %(url)s")
+                    % locals())
+        nova = None
+        try:
+            nova = novaclient.OpenStack(zone.username, zone.password, url)
+            nova.authenticate()
+        except novaclient.exceptions.BadRequest, e:
+            raise exception.NotAuthorized(_("Bad credentials attempting "
+                            "to talk to zone at %(url)s.") % locals())
+
+        nova.servers.create(name, image_id, flavor_id, ipgroup, meta, files,
+                            child_blob)
+
+    def _provision_resource_from_blob(self, context, item, instance_id,
+                                          request_spec, kwargs):
+        """Create the requested resource locally or in a child zone
+           based on what is stored in the zone blob info.
+
+           Attempt to decrypt the blob to see if this request is:
+           1. valid, and
+           2. intended for this zone or a child zone.
+
+           Note: If we have "blob" that means the request was passed
+           into us from a parent zone. If we have "child_blob" that
+           means we gathered the info from one of our children.
+           It's possible that, when we decrypt the 'blob' field, it
+           contains "child_blob" data. In which case we forward the
+           request."""
+
+        host_info = None
+        if "blob" in item:
+            # Request was passed in from above. Is it for us?
+            host_info = self._decrypt_blob(item['blob'])
+        elif "child_blob" in item:
+            # Our immediate child zone provided this info ...
+            host_info = item
+
+        if not host_info:
+            raise InvalidBlob()
+
+        # Valid data ... is it for us?
+        if 'child_zone' in host_info and 'child_blob' in host_info:
+            self._ask_child_zone_to_create_instance(context, host_info,
+                                                    request_spec, kwargs)
+        else:
+            self._provision_resource_locally(context, host_info,
+                                             instance_id, kwargs)
+
+    def _provision_resource(self, context, item, instance_id, request_spec,
+                           kwargs):
+        """Create the requested resource in this Zone or a child zone."""
+        if "hostname" in item:
+            self._provision_resource_locally(context, item, instance_id,
+                            kwargs)
+            return
+
+        self._provision_resource_from_blob(context, item, instance_id,
+                                               request_spec, kwargs)
+
     def schedule_run_instance(self, context, instance_id, request_spec,
                               *args, **kwargs):
         """This method is called from nova.compute.api to provision
@@ -51,8 +171,10 @@ class ZoneAwareScheduler(driver.Scheduler):
 
         # TODO(sandy): We'll have to look for richer specs at some point.
 
-        if 'blob' in request_spec:
+        blob = request_spec.get('blob')
-            self.provision_resource(context, request_spec, instance_id, kwargs)
+        if blob:
+            self._provision_resource(context, request_spec, instance_id,
+                                    request_spec, kwargs)
             return None
 
         # Create build plan and provision ...
@@ -61,28 +183,13 @@ class ZoneAwareScheduler(driver.Scheduler):
             raise driver.NoValidHost(_('No hosts were available'))
 
         for item in build_plan:
-            self.provision_resource(context, item, instance_id, kwargs)
+            self._provision_resource(context, item, instance_id, request_spec,
+                                    kwargs)
 
         # Returning None short-circuits the routing to Compute (since
         # we've already done it here)
         return None
 
-    def provision_resource(self, context, item, instance_id, kwargs):
-        """Create the requested resource in this Zone or a child zone."""
-        if "hostname" in item:
-            host = item['hostname']
-            kwargs['instance_id'] = instance_id
-            rpc.cast(context,
-                     db.queue_get_for(context, "compute", host),
-                     {"method": "run_instance",
-                      "args": kwargs})
-            LOG.debug(_("Casted to compute %(host)s for run_instance")
-                                % locals())
-        else:
-            # TODO(sandy) Provision in child zone ...
-            LOG.warning(_("Provision to Child Zone not supported (yet)"))
-            pass
-
     def select(self, context, request_spec, *args, **kwargs):
         """Select returns a list of weights and zone/host information
         corresponding to the best hosts to service the request. Any
@@ -124,15 +231,15 @@ class ZoneAwareScheduler(driver.Scheduler):
         weighted = self.weigh_hosts(num_instances, request_spec, host_list)
 
         # Next, tack on the best weights from the child zones ...
+        json_spec = json.dumps(request_spec)
         child_results = self._call_zone_method(context, "select",
-                specs=request_spec)
+                specs=json_spec)
         for child_zone, result in child_results:
             for weighting in result:
                 # Remember the child_zone so we can get back to
                 # it later if needed. This implicitly builds a zone
                 # path structure.
-                host_dict = {
+                host_dict = {"weight": weighting["weight"],
-                        "weight": weighting["weight"],
                              "child_zone": child_zone,
                              "child_blob": weighting["blob"]}
                 weighted.append(host_dict)

nova/tests/scheduler/test_zone_aware_scheduler.py

@@ -16,6 +16,7 @@
 Tests For Zone Aware Scheduler.
 """
 
+from nova import exception
 from nova import test
 from nova.scheduler import driver
 from nova.scheduler import zone_aware_scheduler
@@ -90,6 +91,41 @@ def fake_empty_call_zone_method(context, method, specs):
     return []
 
 
+# Hmm, I should probably be using mox for this.
+was_called = False
+
+
+def fake_provision_resource(context, item, instance_id, request_spec, kwargs):
+    global was_called
+    was_called = True
+
+
+def fake_ask_child_zone_to_create_instance(context, zone_info,
+                                           request_spec, kwargs):
+    global was_called
+    was_called = True
+
+
+def fake_provision_resource_locally(context, item, instance_id, kwargs):
+    global was_called
+    was_called = True
+
+
+def fake_provision_resource_from_blob(context, item, instance_id,
+                                      request_spec, kwargs):
+    global was_called
+    was_called = True
+
+
+def fake_decrypt_blob_returns_local_info(blob):
+    return {'foo': True}  # values aren't important.
+
+
+def fake_decrypt_blob_returns_child_info(blob):
+    return {'child_zone': True,
+            'child_blob': True}  # values aren't important. Keys are.
+
+
 def fake_call_zone_method(context, method, specs):
     return [
         ('zone1', [
@@ -149,4 +185,112 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
         fake_context = {}
         self.assertRaises(driver.NoValidHost, sched.schedule_run_instance,
                           fake_context, 1,
-                          dict(host_filter=None, instance_type={}))
+                          dict(host_filter=None,
+                               request_spec={'instance_type': {}}))
+
+    def test_schedule_do_not_schedule_with_hint(self):
+        """
+        Check the local/child zone routing in the run_instance() call.
+        If the zone_blob hint was passed in, don't re-schedule.
+        """
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_provision_resource', fake_provision_resource)
+        request_spec = {
+                'instance_properties': {},
+                'instance_type': {},
+                'filter_driver': 'nova.scheduler.host_filter.AllHostsFilter',
+                'blob': "Non-None blob data"
+            }
+
+        result = sched.schedule_run_instance(None, 1, request_spec)
+        self.assertEquals(None, result)
+        self.assertTrue(was_called)
+
+    def test_provision_resource_local(self):
+        """Provision a resource locally or remotely."""
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_provision_resource_locally',
+                       fake_provision_resource_locally)
+
+        request_spec = {'hostname': "foo"}
+        sched._provision_resource(None, request_spec, 1, request_spec, {})
+        self.assertTrue(was_called)
+
+    def test_provision_resource_remote(self):
+        """Provision a resource locally or remotely."""
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_provision_resource_from_blob',
+                       fake_provision_resource_from_blob)
+
+        request_spec = {}
+        sched._provision_resource(None, request_spec, 1, request_spec, {})
+        self.assertTrue(was_called)
+
+    def test_provision_resource_from_blob_empty(self):
+        """Provision a resource locally or remotely given no hints."""
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        request_spec = {}
+        self.assertRaises(zone_aware_scheduler.InvalidBlob,
+                          sched._provision_resource_from_blob,
+                          None, {}, 1, {}, {})
+
+    def test_provision_resource_from_blob_with_local_blob(self):
+        """
+        Provision a resource locally or remotely when blob hint passed in.
+        """
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_decrypt_blob',
+                       fake_decrypt_blob_returns_local_info)
+        self.stubs.Set(sched, '_provision_resource_locally',
+                       fake_provision_resource_locally)
+
+        request_spec = {'blob': "Non-None blob data"}
+
+        sched._provision_resource_from_blob(None, request_spec, 1,
+                                            request_spec, {})
+        self.assertTrue(was_called)
+
+    def test_provision_resource_from_blob_with_child_blob(self):
+        """
+        Provision a resource locally or remotely when child blob hint
+        passed in.
+        """
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        self.stubs.Set(sched, '_decrypt_blob',
+                       fake_decrypt_blob_returns_child_info)
+        was_called = False
+        self.stubs.Set(sched, '_ask_child_zone_to_create_instance',
+                       fake_ask_child_zone_to_create_instance)
+
+        request_spec = {'blob': "Non-None blob data"}
+
+        sched._provision_resource_from_blob(None, request_spec, 1,
+                                            request_spec, {})
+        self.assertTrue(was_called)
+
+    def test_provision_resource_from_blob_with_immediate_child_blob(self):
+        """
+        Provision a resource locally or remotely when blob hint passed in
+        from an immediate child.
+        """
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_ask_child_zone_to_create_instance',
+                       fake_ask_child_zone_to_create_instance)
+
+        request_spec = {'child_blob': True, 'child_zone': True}
+
+        sched._provision_resource_from_blob(None, request_spec, 1,
+                                            request_spec, {})
+        self.assertTrue(was_called)

nova/tests/test_vmwareapi.py

@@ -69,7 +69,7 @@ class VMWareAPIVMTestCase(test.TestCase):
                   'instance_type': 'm1.large',
                   'mac_address': 'aa:bb:cc:dd:ee:ff',
                   }
-        self.instance = db.instance_create(values)
+        self.instance = db.instance_create(None, values)
 
     def _create_vm(self):
         """Create and spawn the VM."""

nova/tests/vmwareapi/db_fakes.py

@@ -52,7 +52,7 @@ def stub_out_db_instance_api(stubs):
             else:
                 raise NotImplementedError()
 
-    def fake_instance_create(values):
+    def fake_instance_create(context, values):
         """Stubs out the db.instance_create method."""
 
         type_data = INSTANCE_TYPES[values['instance_type']]