From 5f442a5f5e5eac1f9de0984ea243eb7414bb5a22 Mon Sep 17 00:00:00 2001
From: Yuriy Taraday <yorik.sar@gmail.com>
Date: Fri, 17 Jun 2011 18:26:31 +0400
Subject: [PATCH 01/23] Made hostname independent from ec2 id. Add generation
 of hostnames based on display name.

---
 nova/tests/test_compute.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index b4ac2dbc..8af2665b 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -128,7 +128,7 @@ class ComputeTestCase(test.TestCase):
         instance_ref = models.Instance()
         instance_ref['id'] = 1
         instance_ref['volumes'] = [vol1, vol2]
-        instance_ref['hostname'] = 'i-00000001'
+        instance_ref['hostname'] = 'hostname-1'
         instance_ref['host'] = 'dummy'
         return instance_ref
 

From dacbb341dce52428d20a4d1fb141383935118e8c Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 01:22:50 -0700
Subject: [PATCH 02/23] Add some resource checking for memory available when
 scheduling Various changes to d-sched to plan for scheduling on different
 topics, which cleans up some of the resource checking. Re-compute weights
 when building more than 1 instance, accounting for resources that would be
 consumed.

---
 nova/scheduler/host_filter.py          | 10 ++-
 nova/scheduler/least_cost.py           | 39 +++++++----
 nova/scheduler/zone_aware_scheduler.py | 92 +++++++++++++++++++++-----
 3 files changed, 106 insertions(+), 35 deletions(-)

diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py
index bd6b2660..818ae4a3 100644
--- a/nova/scheduler/host_filter.py
+++ b/nova/scheduler/host_filter.py
@@ -305,8 +305,11 @@ class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler):
                     'instance_type': <InstanceType dict>}
     """
 
-    def filter_hosts(self, num, request_spec):
+    def filter_hosts(self, topic, request_spec, hosts):
         """Filter the full host list (from the ZoneManager)"""
+
+        if hosts:
+            return hosts
         filter_name = request_spec.get('filter', None)
         host_filter = choose_host_filter(filter_name)
 
@@ -317,8 +320,9 @@ class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler):
         name, query = host_filter.instance_type_to_filter(instance_type)
         return host_filter.filter_hosts(self.zone_manager, query)
 
-    def weigh_hosts(self, num, request_spec, hosts):
+    def weigh_hosts(self, topic, request_spec, hosts):
         """Derived classes must override this method and return
         a lists of hosts in [{weight, hostname}] format.
         """
-        return [dict(weight=1, hostname=host) for host, caps in hosts]
+        return [dict(weight=1, hostname=hostname, capabilities=caps)
+                for hostname, caps in hosts]
diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py
index 629fe2e4..72db2fd1 100644
--- a/nova/scheduler/least_cost.py
+++ b/nova/scheduler/least_cost.py
@@ -48,25 +48,36 @@ def noop_cost_fn(host):
     return 1
 
 
-flags.DEFINE_integer('fill_first_cost_fn_weight', 1,
+flags.DEFINE_integer('compute_fill_first_cost_fn_weight', 1,
                      'How much weight to give the fill-first cost function')
 
 
-def fill_first_cost_fn(host):
+def compute_fill_first_cost_fn(host):
     """Prefer hosts that have less ram available, filter_hosts will exclude
     hosts that don't have enough ram"""
     hostname, caps = host
-    free_mem = caps['compute']['host_memory_free']
+    free_mem = caps['host_memory_free']
     return free_mem
 
 
 class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler):
-    def get_cost_fns(self):
+    def __init__(self, *args, **kwargs):
+        self.cost_fns_cache = {}
+        super(LeastCoastScheduler, self).__init__(*args, **kwargs)
+
+    def get_cost_fns(self, topic):
         """Returns a list of tuples containing weights and cost functions to
         use for weighing hosts
         """
+
+        if topic in self.cost_fns_cache:
+            return self.cost_fns_cache[topic]
+
         cost_fns = []
         for cost_fn_str in FLAGS.least_cost_scheduler_cost_functions:
+            if not cost_fn_str.startswith('%s_' % topic) and \
+                    not cost_fn_str.startswith('noop'):
+                continue
 
             try:
                 # NOTE(sirp): import_class is somewhat misnamed since it can
@@ -84,23 +95,23 @@ class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler):
 
             cost_fns.append((weight, cost_fn))
 
+        self.cost_fns_cache[topic] = cost_fns
         return cost_fns
 
-    def weigh_hosts(self, num, request_spec, hosts):
+    def weigh_hosts(self, topic, request_spec, hosts):
         """Returns a list of dictionaries of form:
-            [ {weight: weight, hostname: hostname} ]"""
+           [ {weight: weight, hostname: hostname, capabilities: capabs} ]
+        """
 
-        # FIXME(sirp): weigh_hosts should handle more than just instances
-        hostnames = [hostname for hostname, caps in hosts]
-
-        cost_fns = self.get_cost_fns()
+        cost_fns = self.get_cost_fns(topic)
         costs = weighted_sum(domain=hosts, weighted_fns=cost_fns)
 
         weighted = []
         weight_log = []
-        for cost, hostname in zip(costs, hostnames):
+        for cost, (hostname, caps) in zip(costs, hosts):
             weight_log.append("%s: %s" % (hostname, "%.2f" % cost))
-            weight_dict = dict(weight=cost, hostname=hostname)
+            weight_dict = dict(weight=cost, hostname=hostname,
+                    capabilities=caps)
             weighted.append(weight_dict)
 
         LOG.debug(_("Weighted Costs => %s") % weight_log)
@@ -127,7 +138,8 @@ def weighted_sum(domain, weighted_fns, normalize=True):
     weighted_fns - list of weights and functions like:
         [(weight, objective-functions)]
 
-    Returns an unsorted of scores. To pair with hosts do: zip(scores, hosts)
+    Returns an unsorted list of scores. To pair with hosts do:
+        zip(scores, hosts)
     """
     # Table of form:
     #   { domain1: [score1, score2, ..., scoreM]
@@ -150,7 +162,6 @@ def weighted_sum(domain, weighted_fns, normalize=True):
     domain_scores = []
     for idx in sorted(score_table):
         elem_score = sum(score_table[idx])
-        elem = domain[idx]
         domain_scores.append(elem_score)
 
     return domain_scores
diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index e7bff2fa..d4d3d041 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -224,18 +224,34 @@ class ZoneAwareScheduler(driver.Scheduler):
             raise NotImplemented(_("Zone Aware Scheduler only understands "
                                    "Compute nodes (for now)"))
 
-        #TODO(sandy): how to infer this from OS API params?
-        num_instances = 1
+        num_instances = request_spec['num_instances']
+        instance_type = request_spec['instance_type']
 
-        # Filter local hosts based on requirements ...
-        host_list = self.filter_hosts(num_instances, request_spec)
+        weighted = []
+        host_list = None
 
-        # TODO(sirp): weigh_hosts should also be a function of 'topic' or
-        # resources, so that we can apply different objective functions to it
+        for i in xrange(num_instances):
+            # Filter local hosts based on requirements ...
+            #
+            # The first pass through here will pass 'None' as the
+            # host_list.. which tells the filter to build the full
+            # list of hosts.
+            # On a 2nd pass, the filter can modify the host_list with
+            # any updates it needs to make based on resources that
+            # may have been consumed from a previous build..
+            host_list = self.filter_hosts(topic, request_spec, host_list)
+            if not host_list:
+                break
 
-        # then weigh the selected hosts.
-        # weighted = [{weight=weight, name=hostname}, ...]
-        weighted = self.weigh_hosts(num_instances, request_spec, host_list)
+            # then weigh the selected hosts.
+            # weighted = [{weight=weight, hostname=hostname,
+            #              capabilities=capabs}, ...]
+            weights = self.weigh_hosts(topic, request_spec, host_list)
+            weights.sort(key=operator.itemgetter('weight'))
+            best_weight = weights[0]
+            weighted.append(best_weight)
+            self.consume_resources(best_weight['capabilities'],
+                    instance_type)
 
         # Next, tack on the best weights from the child zones ...
         json_spec = json.dumps(request_spec)
@@ -254,18 +270,58 @@ class ZoneAwareScheduler(driver.Scheduler):
         weighted.sort(key=operator.itemgetter('weight'))
         return weighted
 
-    def filter_hosts(self, num, request_spec):
-        """Derived classes must override this method and return
-           a list of hosts in [(hostname, capability_dict)] format.
+    def compute_filter(self, hostname, capabilities, request_spec):
+        """Return whether or not we can schedule to this compute node.
+        Derived classes should override this and return True if the host
+        is acceptable for scheduling.
         """
-        # NOTE(sirp): The default logic is the equivalent to AllHostsFilter
-        service_states = self.zone_manager.service_states
-        return [(host, services)
-                for host, services in service_states.iteritems()]
+        instance_type = request_spec['instance_type']
+        reqested_mem = instance_type['memory_mb']
+        return capabilities['host_memory_free'] >= requested_mem
 
-    def weigh_hosts(self, num, request_spec, hosts):
+    def filter_hosts(self, topic, request_spec, host_list=None):
+        """Return a list of hosts which are acceptable for scheduling.
+        Return value should be a list of (hostname, capability_dict)s.
+        Derived classes may override this, but may find the
+        '<topic>_filter' function more appropriate.
+        """
+
+        def _default_filter(self, hostname, capabilities, request_spec):
+            """Default filter function if there's no <topic>_filter"""
+            # NOTE(sirp): The default logic is the equivalent to
+            # AllHostsFilter
+            return True
+
+        filter_func = getattr(self, '%s_filter' % topic, _default_filter)
+
+        filtered_hosts = []
+        if host_list is None:
+            host_list = self.zone_manager.service_states.iteritems()
+        for host, services in host_list:
+            if topic not in services:
+                continue
+            if filter_func(host, services['topic'], request_spec):
+                filtered_hosts.append((host, services['topic']))
+
+    def weigh_hosts(self, topic, request_spec, hosts):
         """Derived classes may override this to provide more sophisticated
         scheduling objectives
         """
         # NOTE(sirp): The default logic is the same as the NoopCostFunction
-        return [dict(weight=1, hostname=host) for host, caps in hosts]
+        return [dict(weight=1, hostname=hostname, capabilities=capabilities)
+                for hostname, capabilities in hosts]
+
+    def compute_consume(self, capabilities, instance_type):
+        """Consume compute resources for selected host"""
+
+        requested_mem = max(instance_type['memory_mb'], 0)
+        capabilities['host_memory_free'] -= requested_mem
+
+    def consume_resources(self, topic, capabilities, instance_type):
+        """Consume resources for a specific host.  'host' is a tuple
+        of the hostname and the services"""
+
+        consume_func = getattr(self, '%s_consume' % topic, None)
+        if not consume_func:
+            return
+        consume_func(capabilities, instance_type)

From bb4fa1ee1aaf2fb2edb6d51d6eae4a28f8fbc95a Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 06:52:19 -0700
Subject: [PATCH 03/23] minor fixups

---
 nova/scheduler/host_filter.py          | 4 +---
 nova/scheduler/zone_aware_scheduler.py | 1 +
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py
index 818ae4a3..b336665b 100644
--- a/nova/scheduler/host_filter.py
+++ b/nova/scheduler/host_filter.py
@@ -305,11 +305,9 @@ class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler):
                     'instance_type': <InstanceType dict>}
     """
 
-    def filter_hosts(self, topic, request_spec, hosts):
+    def filter_hosts(self, topic, request_spec, hosts=None):
         """Filter the full host list (from the ZoneManager)"""
 
-        if hosts:
-            return hosts
         filter_name = request_spec.get('filter', None)
         host_filter = choose_host_filter(filter_name)
 
diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index d4d3d041..70cb83e8 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -302,6 +302,7 @@ class ZoneAwareScheduler(driver.Scheduler):
                 continue
             if filter_func(host, services['topic'], request_spec):
                 filtered_hosts.append((host, services['topic']))
+        return filtered_hosts
 
     def weigh_hosts(self, topic, request_spec, hosts):
         """Derived classes may override this to provide more sophisticated

From bc23101b7a33d98afe7b354cba9b4b88e10c6e68 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 09:43:27 -0700
Subject: [PATCH 04/23] an int() was missed being removed from UUID changes
 when zone rerouting kicks in

---
 nova/scheduler/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py
index 1bb047e2..0aed7568 100644
--- a/nova/scheduler/api.py
+++ b/nova/scheduler/api.py
@@ -169,7 +169,7 @@ def _issue_novaclient_command(nova, zone, collection, method_name, item_id):
     result = None
     try:
         try:
-            result = manager.get(int(item_id))
+            result = manager.get(item_id)
         except ValueError, e:
             result = manager.find(name=item_id)
     except novaclient.NotFound:

From 63f22bfe684d00755c1d5589dd602981166a2e85 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 21:59:54 -0700
Subject: [PATCH 05/23] missed passing an argument to consume_resources

---
 nova/scheduler/zone_aware_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index 70cb83e8..073bdd3b 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -250,7 +250,7 @@ class ZoneAwareScheduler(driver.Scheduler):
             weights.sort(key=operator.itemgetter('weight'))
             best_weight = weights[0]
             weighted.append(best_weight)
-            self.consume_resources(best_weight['capabilities'],
+            self.consume_resources(topic, best_weight['capabilities'],
                     instance_type)
 
         # Next, tack on the best weights from the child zones ...

From 47b4f75c43709f7cdd202e994a04f076b40fa889 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 22:48:44 -0700
Subject: [PATCH 06/23] debug logging of number of instances to build in
 scheduler

---
 nova/scheduler/zone_aware_scheduler.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index 073bdd3b..a747526d 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -180,18 +180,21 @@ class ZoneAwareScheduler(driver.Scheduler):
                                     request_spec, kwargs)
             return None
 
+        num_instances = request_spec['num_instances']
+        LOG.debug(_("Attemping to build %d instance%s") %
+            (num_instances, "" if num_instances == 1 else "s"))
+
         # Create build plan and provision ...
         build_plan = self.select(context, request_spec)
         if not build_plan:
             raise driver.NoValidHost(_('No hosts were available'))
 
-        for num in xrange(request_spec['num_instances']):
+        for num in xrange(num_instances):
             if not build_plan:
                 break
-
             item = build_plan.pop(0)
-            self._provision_resource(context, item, instance_id, request_spec,
-                                    kwargs)
+            self._provision_resource(context, item, instance_id,
+                    request_spec, kwargs)
 
         # Returning None short-circuits the routing to Compute (since
         # we've already done it here)

From c24dcbc30b931cfea2ddf9e1968b16ef0c7df3a0 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 22:55:45 -0700
Subject: [PATCH 07/23] typo in least cost scheduler

---
 nova/scheduler/least_cost.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py
index 72db2fd1..9376631e 100644
--- a/nova/scheduler/least_cost.py
+++ b/nova/scheduler/least_cost.py
@@ -63,7 +63,7 @@ def compute_fill_first_cost_fn(host):
 class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler):
     def __init__(self, *args, **kwargs):
         self.cost_fns_cache = {}
-        super(LeastCoastScheduler, self).__init__(*args, **kwargs)
+        super(LeastCostScheduler, self).__init__(*args, **kwargs)
 
     def get_cost_fns(self, topic):
         """Returns a list of tuples containing weights and cost functions to

From 8ea7a715d7fd8390c00e702b7cfdba72bdaf08eb Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 23:00:15 -0700
Subject: [PATCH 08/23] more typos

---
 nova/scheduler/zone_aware_scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index a747526d..8f218c15 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -303,8 +303,8 @@ class ZoneAwareScheduler(driver.Scheduler):
         for host, services in host_list:
             if topic not in services:
                 continue
-            if filter_func(host, services['topic'], request_spec):
-                filtered_hosts.append((host, services['topic']))
+            if filter_func(host, services[topic], request_spec):
+                filtered_hosts.append((host, services[topic]))
         return filtered_hosts
 
     def weigh_hosts(self, topic, request_spec, hosts):

From b07544045f06ef2ada44b3bbae5d84dea5b33ed9 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 23:05:12 -0700
Subject: [PATCH 09/23] requested_mem typo

---
 nova/scheduler/zone_aware_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index 8f218c15..769b2dd0 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -279,7 +279,7 @@ class ZoneAwareScheduler(driver.Scheduler):
         is acceptable for scheduling.
         """
         instance_type = request_spec['instance_type']
-        reqested_mem = instance_type['memory_mb']
+        requested_mem = instance_type['memory_mb']
         return capabilities['host_memory_free'] >= requested_mem
 
     def filter_hosts(self, topic, request_spec, host_list=None):

From 25c3ae3534debf4be4455d49e8aeba1b582b0150 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Thu, 23 Jun 2011 23:38:32 -0700
Subject: [PATCH 10/23] LeastCostScheduler wasn't checking for topic cost
 functions correctly. Added support so that
 --least_cost_scheduler_cost_functions only needs to have method names
 specified, instead of the full blown version with module and class name. 
 Still works the old way, too.

---
 nova/scheduler/least_cost.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py
index 9376631e..6f5eb66f 100644
--- a/nova/scheduler/least_cost.py
+++ b/nova/scheduler/least_cost.py
@@ -75,8 +75,15 @@ class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler):
 
         cost_fns = []
         for cost_fn_str in FLAGS.least_cost_scheduler_cost_functions:
-            if not cost_fn_str.startswith('%s_' % topic) and \
-                    not cost_fn_str.startswith('noop'):
+            if '.' in cost_fn_str:
+                short_name = cost_fn_str.split('.')[-1]
+            else:
+                short_name = cost_fn_str
+                cost_fn_str = "%s.%s.%s" % (
+                        __name__, self.__class__.__name__, short_name)
+
+            if not (short_name.startswith('%s_' % topic) or
+                    short_name.startswith('noop')):
                 continue
 
             try:

From 26348f56fcd1ed3bb980c6b051636b12106f2197 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Fri, 24 Jun 2011 00:26:55 -0700
Subject: [PATCH 11/23] on 2nd run through filter_hosts, we've already
 accounted for the topic memory needs converted to Bytes from MB

---
 nova/scheduler/zone_aware_scheduler.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index 769b2dd0..e6383b20 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -244,6 +244,8 @@ class ZoneAwareScheduler(driver.Scheduler):
             # may have been consumed from a previous build..
             host_list = self.filter_hosts(topic, request_spec, host_list)
             if not host_list:
+                LOG.warn(_("Ran out of available hosts after weighing "
+                        "%d of %d instances") % (i, num_instances))
                 break
 
             # then weigh the selected hosts.
@@ -279,7 +281,7 @@ class ZoneAwareScheduler(driver.Scheduler):
         is acceptable for scheduling.
         """
         instance_type = request_spec['instance_type']
-        requested_mem = instance_type['memory_mb']
+        requested_mem = instance_type['memory_mb'] * 1024 * 1024
         return capabilities['host_memory_free'] >= requested_mem
 
     def filter_hosts(self, topic, request_spec, host_list=None):
@@ -297,14 +299,20 @@ class ZoneAwareScheduler(driver.Scheduler):
 
         filter_func = getattr(self, '%s_filter' % topic, _default_filter)
 
-        filtered_hosts = []
         if host_list is None:
+            first_run = True
             host_list = self.zone_manager.service_states.iteritems()
+        else:
+            first_run = False
+
+        filtered_hosts = []
         for host, services in host_list:
-            if topic not in services:
-                continue
-            if filter_func(host, services[topic], request_spec):
-                filtered_hosts.append((host, services[topic]))
+            if first_run:
+                if topic not in services:
+                    continue
+                services = services['topic']
+            if filter_func(host, services, request_spec):
+                filtered_hosts.append((host, services))
         return filtered_hosts
 
     def weigh_hosts(self, topic, request_spec, hosts):
@@ -318,7 +326,7 @@ class ZoneAwareScheduler(driver.Scheduler):
     def compute_consume(self, capabilities, instance_type):
         """Consume compute resources for selected host"""
 
-        requested_mem = max(instance_type['memory_mb'], 0)
+        requested_mem = max(instance_type['memory_mb'], 0) * 1024 * 1024
         capabilities['host_memory_free'] -= requested_mem
 
     def consume_resources(self, topic, capabilities, instance_type):

From 369d79dc66024b19287748838b92e168d85bebb5 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Fri, 24 Jun 2011 00:30:58 -0700
Subject: [PATCH 12/23] same typo i made before!

---
 nova/scheduler/zone_aware_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index e6383b20..e24c2256 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -310,7 +310,7 @@ class ZoneAwareScheduler(driver.Scheduler):
             if first_run:
                 if topic not in services:
                     continue
-                services = services['topic']
+                services = services[topic]
             if filter_func(host, services, request_spec):
                 filtered_hosts.append((host, services))
         return filtered_hosts

From 3520d508486305efe082b2831d23e559c69f7c90 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Mon, 27 Jun 2011 15:05:37 -0700
Subject: [PATCH 13/23] Made _issue_novaclient_command() behave better. Fixed a
 bunch of tests.

---
 nova/scheduler/api.py                         | 49 +++++++++++++------
 nova/scheduler/zone_aware_scheduler.py        |  4 +-
 .../scheduler/test_least_cost_scheduler.py    | 11 +++--
 .../scheduler/test_zone_aware_scheduler.py    | 35 ++++++-------
 4 files changed, 58 insertions(+), 41 deletions(-)

diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py
index 0aed7568..5d62beb8 100644
--- a/nova/scheduler/api.py
+++ b/nova/scheduler/api.py
@@ -162,32 +162,53 @@ def child_zone_helper(zone_list, func):
                     _wrap_method(_process, func), zone_list)]
 
 
-def _issue_novaclient_command(nova, zone, collection, method_name, item_id):
+def _issue_novaclient_command(nova, zone, collection,
+        method_name, *args, **kwargs):
     """Use novaclient to issue command to a single child zone.
-       One of these will be run in parallel for each child zone."""
+       One of these will be run in parallel for each child zone.
+    """
     manager = getattr(nova, collection)
-    result = None
-    try:
+
+    # NOTE(comstud): This is not ideal, but we have to do this based on
+    # how novaclient is implemented right now.
+    # 'find' is special cased as novaclient requires kwargs for it to
+    # filter on a 'get_all'.
+    # Every other method first needs to do a 'get' on the first argument
+    # passed, which should be a UUID.  If it's 'get' itself that we want,
+    # we just return the result.  Otherwise, we next call the real method
+    # that's wanted... passing other arguments that may or may not exist.
+    if method_name in ['find', 'findall']:
         try:
-            result = manager.get(item_id)
-        except ValueError, e:
-            result = manager.find(name=item_id)
+            return getattr(manager, method_name)(**kwargs)
+        except novaclient.NotFound:
+            url = zone.api_url
+            LOG.debug(_("%(collection)s.%(method_name)s didn't find "
+                    "anything matching '%(kwargs)s' on '%(url)s'" %
+                    locals()))
+            return None
+
+    args = list(args)
+    # pop off the UUID to look up
+    item = args.pop(0)
+    try:
+        result = manager.get(item)
     except novaclient.NotFound:
         url = zone.api_url
-        LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" %
+        LOG.debug(_("%(collection)s '%(item)s' not found on '%(url)s'" %
                                                 locals()))
         return None
 
-    if method_name.lower() not in ['get', 'find']:
-        result = getattr(result, method_name)()
+    if method_name.lower() != 'get':
+        # if we're doing something other than 'get', call it passing args.
+        result = getattr(result, method_name)(*args, **kwargs)
     return result
 
 
-def wrap_novaclient_function(f, collection, method_name, item_id):
-    """Appends collection, method_name and item_id to the incoming
+def wrap_novaclient_function(f, collection, method_name, *args, **kwargs):
+    """Appends collection, method_name and arguments to the incoming
     (nova, zone) call from child_zone_helper."""
     def inner(nova, zone):
-        return f(nova, zone, collection, method_name, item_id)
+        return f(nova, zone, collection, method_name, *args, **kwargs)
 
     return inner
 
@@ -220,7 +241,7 @@ class reroute_compute(object):
            the wrapped method. (This ensures that zone-local code can
            continue to use integer IDs).
 
-        4. If the item was not found, we delgate the call to a child zone
+        4. If the item was not found, we delegate the call to a child zone
            using the UUID.
     """
     def __init__(self, method_name):
diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index e24c2256..eb116fac 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -180,7 +180,7 @@ class ZoneAwareScheduler(driver.Scheduler):
                                     request_spec, kwargs)
             return None
 
-        num_instances = request_spec['num_instances']
+        num_instances = request_spec.get('num_instances', 1)
         LOG.debug(_("Attemping to build %d instance%s") %
             (num_instances, "" if num_instances == 1 else "s"))
 
@@ -227,7 +227,7 @@ class ZoneAwareScheduler(driver.Scheduler):
             raise NotImplemented(_("Zone Aware Scheduler only understands "
                                    "Compute nodes (for now)"))
 
-        num_instances = request_spec['num_instances']
+        num_instances = request_spec.get('num_instances', 1)
         instance_type = request_spec['instance_type']
 
         weighted = []
diff --git a/nova/tests/scheduler/test_least_cost_scheduler.py b/nova/tests/scheduler/test_least_cost_scheduler.py
index 9a5318ae..49791053 100644
--- a/nova/tests/scheduler/test_least_cost_scheduler.py
+++ b/nova/tests/scheduler/test_least_cost_scheduler.py
@@ -122,15 +122,16 @@ class LeastCostSchedulerTestCase(test.TestCase):
                     for hostname, caps in hosts]
         self.assertWeights(expected, num, request_spec, hosts)
 
-    def test_fill_first_cost_fn(self):
+    def test_compute_fill_first_cost_fn(self):
         FLAGS.least_cost_scheduler_cost_functions = [
-            'nova.scheduler.least_cost.fill_first_cost_fn',
+            'nova.scheduler.least_cost.compute_fill_first_cost_fn',
         ]
-        FLAGS.fill_first_cost_fn_weight = 1
+        FLAGS.compute_fill_first_cost_fn_weight = 1
 
         num = 1
-        request_spec = {}
-        hosts = self.sched.filter_hosts(num, request_spec)
+        instance_type = {'memory_mb': 1024}
+        request_spec = {'instance_type': instance_type}
+        hosts = self.sched.filter_hosts('compute', request_spec, None)
 
         expected = []
         for idx, (hostname, caps) in enumerate(hosts):
diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py
index 37c6488c..b2599f1b 100644
--- a/nova/tests/scheduler/test_zone_aware_scheduler.py
+++ b/nova/tests/scheduler/test_zone_aware_scheduler.py
@@ -55,29 +55,21 @@ def fake_zone_manager_service_states(num_hosts):
 
 
 class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler):
-    def filter_hosts(self, num, specs):
-        # NOTE(sirp): this is returning [(hostname, services)]
-        return self.zone_manager.service_states.items()
-
-    def weigh_hosts(self, num, specs, hosts):
-        fake_weight = 99
-        weighted = []
-        for hostname, caps in hosts:
-            weighted.append(dict(weight=fake_weight, name=hostname))
-        return weighted
+    # No need to stub anything at the moment
+    pass
 
 
 class FakeZoneManager(zone_manager.ZoneManager):
     def __init__(self):
         self.service_states = {
             'host1': {
-                'compute': {'ram': 1000},
+                'compute': {'host_memory_free': 1000*1024*1024},
             },
             'host2': {
-                'compute': {'ram': 2000},
+                'compute': {'host_memory_free': 2000*1024*1024},
             },
             'host3': {
-                'compute': {'ram': 3000},
+                'compute': {'host_memory_free': 3000*1024*1024},
             },
         }
 
@@ -164,13 +156,17 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
         sched.set_zone_manager(zm)
 
         fake_context = {}
-        build_plan = sched.select(fake_context, {})
+        build_plan = sched.select(fake_context,
+                {'instance_type': {'memory_mb': 512},
+                    'num_instances': 4 })
 
-        self.assertEqual(15, len(build_plan))
+        # 4 from local zones, 12 from remotes
+        self.assertEqual(16, len(build_plan))
 
-        hostnames = [plan_item['name']
-                     for plan_item in build_plan if 'name' in plan_item]
-        self.assertEqual(3, len(hostnames))
+        hostnames = [plan_item['hostname']
+                     for plan_item in build_plan if 'hostname' in plan_item]
+        # 4 local hosts
+        self.assertEqual(4, len(hostnames))
 
     def test_empty_zone_aware_scheduler(self):
         """
@@ -185,8 +181,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
         fake_context = {}
         self.assertRaises(driver.NoValidHost, sched.schedule_run_instance,
                           fake_context, 1,
-                          dict(host_filter=None,
-                               request_spec={'instance_type': {}}))
+                          dict(host_filter=None, instance_type={}))
 
     def test_schedule_do_not_schedule_with_hint(self):
         """

From 2602234e0fed374c2d619741931245e607c269cf Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Mon, 27 Jun 2011 15:17:19 -0700
Subject: [PATCH 14/23] logging fixes

---
 nova/scheduler/zone_aware_scheduler.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index eb116fac..638072ee 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -181,8 +181,7 @@ class ZoneAwareScheduler(driver.Scheduler):
             return None
 
         num_instances = request_spec.get('num_instances', 1)
-        LOG.debug(_("Attemping to build %d instance%s") %
-            (num_instances, "" if num_instances == 1 else "s"))
+        LOG.debug(_("Attemping to build %d instance(s)") % locals())
 
         # Create build plan and provision ...
         build_plan = self.select(context, request_spec)
@@ -245,7 +244,7 @@ class ZoneAwareScheduler(driver.Scheduler):
             host_list = self.filter_hosts(topic, request_spec, host_list)
             if not host_list:
                 LOG.warn(_("Ran out of available hosts after weighing "
-                        "%d of %d instances") % (i, num_instances))
+                        "%(i)d of %(num_instances)d instances") % locals())
                 break
 
             # then weigh the selected hosts.

From 6fae4b4e551d612408854e52338fb788d50f0a1b Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Tue, 28 Jun 2011 08:08:13 -0700
Subject: [PATCH 15/23] log formatting typo pep8 fixes

---
 nova/scheduler/zone_aware_scheduler.py            | 3 ++-
 nova/tests/scheduler/test_zone_aware_scheduler.py | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index 638072ee..2efad7bf 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -181,7 +181,8 @@ class ZoneAwareScheduler(driver.Scheduler):
             return None
 
         num_instances = request_spec.get('num_instances', 1)
-        LOG.debug(_("Attemping to build %d instance(s)") % locals())
+        LOG.debug(_("Attempting to build %(num_instances)d instance(s)") %
+                locals())
 
         # Create build plan and provision ...
         build_plan = self.select(context, request_spec)
diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py
index b2599f1b..1e23e3ee 100644
--- a/nova/tests/scheduler/test_zone_aware_scheduler.py
+++ b/nova/tests/scheduler/test_zone_aware_scheduler.py
@@ -63,13 +63,13 @@ class FakeZoneManager(zone_manager.ZoneManager):
     def __init__(self):
         self.service_states = {
             'host1': {
-                'compute': {'host_memory_free': 1000*1024*1024},
+                'compute': {'host_memory_free': 1073741824},
             },
             'host2': {
-                'compute': {'host_memory_free': 2000*1024*1024},
+                'compute': {'host_memory_free': 2147483648},
             },
             'host3': {
-                'compute': {'host_memory_free': 3000*1024*1024},
+                'compute': {'host_memory_free': 3221225472},
             },
         }
 
@@ -158,7 +158,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
         fake_context = {}
         build_plan = sched.select(fake_context,
                 {'instance_type': {'memory_mb': 512},
-                    'num_instances': 4 })
+                    'num_instances': 4})
 
         # 4 from local zones, 12 from remotes
         self.assertEqual(16, len(build_plan))

From 8ae8892fc9d07dba91d47915700b086b9a7b97ca Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Tue, 28 Jun 2011 08:12:08 -0700
Subject: [PATCH 16/23] update a test docstring to make it clear we're testing
 multiple instance builds

---
 nova/tests/scheduler/test_zone_aware_scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py
index 1e23e3ee..32f5150a 100644
--- a/nova/tests/scheduler/test_zone_aware_scheduler.py
+++ b/nova/tests/scheduler/test_zone_aware_scheduler.py
@@ -146,8 +146,8 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
 
     def test_zone_aware_scheduler(self):
         """
-        Create a nested set of FakeZones, ensure that a select call returns the
-        appropriate build plan.
+        Create a nested set of FakeZones, try to build multiple instances
+        and ensure that a select call returns the appropriate build plan.
         """
         sched = FakeZoneAwareScheduler()
         self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method)

From 393f03377a61167441462334ae603949aaabc2d5 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Tue, 28 Jun 2011 08:53:13 -0700
Subject: [PATCH 17/23] change variable names to remove future conflict with
 sandy's zone-offsets branch

---
 nova/scheduler/zone_aware_scheduler.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index 2efad7bf..2e6662a7 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -192,9 +192,10 @@ class ZoneAwareScheduler(driver.Scheduler):
         for num in xrange(num_instances):
             if not build_plan:
                 break
-            item = build_plan.pop(0)
-            self._provision_resource(context, item, instance_id,
-                    request_spec, kwargs)
+
+            build_plan_item = build_plan.pop(0)
+            self._provision_resource(context, build_plan_item, instance_id,
+                                     request_spec, kwargs)
 
         # Returning None short-circuits the routing to Compute (since
         # we've already done it here)

From 98d60848b9ac61bae1079065537bece6bf367018 Mon Sep 17 00:00:00 2001
From: Yuriy Taraday <yorik.sar@gmail.com>
Date: Wed, 29 Jun 2011 13:37:24 +0400
Subject: [PATCH 18/23] Add test for hostname generation.

---
 nova/tests/test_compute.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index 8af2665b..11ae7403 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -160,6 +160,18 @@ class ComputeTestCase(test.TestCase):
             db.security_group_destroy(self.context, group['id'])
             db.instance_destroy(self.context, ref[0]['id'])
 
+    def test_default_hostname_generator(self):
+        cases = [(None, 'server_1'), ('Hello, Server!', 'hello_server'),
+                 ('<}\x1fh\x10e\x08l\x02l\x05o\x12!{>', 'hello')]
+        for display_name, hostname in cases:
+            ref = self.compute_api.create(self.context,
+                instance_types.get_default_instance_type(), None,
+                display_name=display_name)
+            try:
+                self.assertEqual(ref[0]['hostname'], hostname)
+            finally:
+                db.instance_destroy(self.context, ref[0]['id'])
+        
     def test_destroy_instance_disassociates_security_groups(self):
         """Make sure destroying disassociates security groups"""
         group = self._create_group()

From 37b14b979b1b76755314d83bfddd8b4e1e628663 Mon Sep 17 00:00:00 2001
From: Thierry Carrez <thierry@openstack.org>
Date: Wed, 29 Jun 2011 16:45:46 +0200
Subject: [PATCH 19/23] Fix nova-manage vm list

---
 bin/nova-manage | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bin/nova-manage b/bin/nova-manage
index 02f20347..d5390b63 100755
--- a/bin/nova-manage
+++ b/bin/nova-manage
@@ -617,7 +617,7 @@ class VmCommands(object):
         :param host: show all instance on specified host.
         :param instance: show specificed instance.
         """
-        print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \
+        print "%-10s %-15s %-10s %-10s %-26s %-9s %-9s %-9s" \
               "  %-10s %-10s %-10s %-5s" % (
             _('instance'),
             _('node'),
@@ -639,14 +639,14 @@ class VmCommands(object):
                            context.get_admin_context(), host)
 
         for instance in instances:
-            print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \
+            print "%-10s %-15s %-10s %-10s %-26s %-9s %-9s %-9s" \
                   "  %-10s %-10s %-10s %-5d" % (
                 instance['hostname'],
                 instance['host'],
-                instance['instance_type'],
+                instance['instance_type'].name,
                 instance['state_description'],
                 instance['launched_at'],
-                instance['image_id'],
+                instance['image_ref'],
                 instance['kernel_id'],
                 instance['ramdisk_id'],
                 instance['project_id'],

From 0277e60aaa470148dda969743788865efb730286 Mon Sep 17 00:00:00 2001
From: Thierry Carrez <thierry@openstack.org>
Date: Wed, 29 Jun 2011 16:52:55 +0200
Subject: [PATCH 20/23] Fix 'undefined name 'e'' pylint error

---
 bin/nova-manage | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/nova-manage b/bin/nova-manage
index d5390b63..51e0c32c 100755
--- a/bin/nova-manage
+++ b/bin/nova-manage
@@ -878,7 +878,7 @@ class InstanceTypeCommands(object):
         try:
             instance_types.create(name, memory, vcpus, local_gb,
                                   flavorid, swap, rxtx_quota, rxtx_cap)
-        except exception.InvalidInput:
+        except exception.InvalidInput, e:
             print "Must supply valid parameters to create instance_type"
             print e
             sys.exit(1)

From d308bab3d8ba170cb6a1f8eb62ace4daf73a915b Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Wed, 29 Jun 2011 09:49:19 -0700
Subject: [PATCH 21/23] Fanout queues use unique queue names, so the consumer
 should have exclusive access.  This means that they also get auto deleted
 when we're done with them, so they're not left around on a service restart. 
 Fixes lp:803165

---
 nova/rpc.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/nova/rpc.py b/nova/rpc.py
index 2e78a31e..9f0b507f 100644
--- a/nova/rpc.py
+++ b/nova/rpc.py
@@ -275,6 +275,11 @@ class FanoutAdapterConsumer(AdapterConsumer):
         unique = uuid.uuid4().hex
         self.queue = '%s_fanout_%s' % (topic, unique)
         self.durable = False
+        # Fanout creates unique queue names, so we should auto-remove
+        # them when done, so they're not left around on restart.
+        # Also, we're the only one that should be consuming.  exclusive
+        # implies auto_delete, so we'll just set that..
+        self.exclusive = True
         LOG.info(_('Created "%(exchange)s" fanout exchange '
                    'with "%(key)s" routing key'),
                  dict(exchange=self.exchange, key=self.routing_key))

From 23f51be9b69a36f080f5c5463e66c6dda3182296 Mon Sep 17 00:00:00 2001
From: Chris Behrens <cbehrens@codestud.com>
Date: Wed, 29 Jun 2011 12:23:26 -0700
Subject: [PATCH 22/23] Fixed indentation issues Fixed min/max_count checking
 issues Fixed a wrongly log message when zone aware scheduler finds no
 suitable hosts

---
 nova/scheduler/zone_aware_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index 2e6662a7..9e4fc47d 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -245,7 +245,7 @@ class ZoneAwareScheduler(driver.Scheduler):
             # may have been consumed from a previous build..
             host_list = self.filter_hosts(topic, request_spec, host_list)
             if not host_list:
-                LOG.warn(_("Ran out of available hosts after weighing "
+                LOG.warn(_("Filter returned no hosts after processing "
                         "%(i)d of %(num_instances)d instances") % locals())
                 break
 

From b60c626ee23bc7a99d62c167f7ba87fb9415364d Mon Sep 17 00:00:00 2001
From: Yuriy Taraday <yorik.sar@gmail.com>
Date: Thu, 30 Jun 2011 15:43:18 +0400
Subject: [PATCH 23/23] PEP8 fix.

---
 nova/tests/test_compute.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index 11ae7403..a53d94d7 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -171,7 +171,7 @@ class ComputeTestCase(test.TestCase):
                 self.assertEqual(ref[0]['hostname'], hostname)
             finally:
                 db.instance_destroy(self.context, ref[0]['id'])
-        
+
     def test_destroy_instance_disassociates_security_groups(self):
         """Make sure destroying disassociates security groups"""
         group = self._create_group()