add InstanceList.get_all_uuids_by_hosts() method

We have an existing InstanceList.get_all_uuids_by_host() (note: single
host) method that returns the instance UUIDs for one host. This patch
adds an InstanceList.get_all_uuids_by_hosts() method that accepts an
iterable of host names and returns a dict, keyed by host name, of the
list of instance UUIDs associated with that host.

In writing this patch, I changed the implementation of the DB API's
_instance_get_all_uuids_by_host method to *not* use the model_query()
helper method and instead just use the sqlalchemy core expression API to
construct the SELECT statement.

Following patches modify the scheduler's host manager to take advantage
of this faster batch retrieval of many host -> instance UUIDs mapping to
avoid doing multiple queries for selected compute nodes in the
scheduler.

Change-Id: If92fe8b75d20a738f37e2a74c52c59bfc699a74f

nova/db/api.py

@@ -793,9 +793,11 @@ def instance_get_all(context, columns_to_join=None):
     return IMPL.instance_get_all(context, columns_to_join=columns_to_join)
 
 
-def instance_get_all_uuids_by_host(context, host):
-    """Get a list of instance uuids on host."""
-    return IMPL.instance_get_all_uuids_by_host(context, host)
+def instance_get_all_uuids_by_hosts(context, hosts):
+    """Get a dict, keyed by hostname, of a list of instance uuids on one or
+    more hosts.
+    """
+    return IMPL.instance_get_all_uuids_by_hosts(context, hosts)
 
 
 def instance_get_all_by_filters(context, filters, sort_key='created_at',

nova/db/sqlalchemy/api.py

@@ -1504,7 +1504,8 @@ def fixed_ip_get_by_instance(context, instance_uuid):
 
 @pick_context_manager_reader
 def fixed_ip_get_by_host(context, host):
-    instance_uuids = _instance_get_all_uuids_by_host(context, host)
+    instance_uuids = _instance_get_all_uuids_by_hosts(
+        context, [host]).get(host, [])
     if not instance_uuids:
         return []
 
@@ -2657,23 +2658,30 @@ def instance_get_all_by_host(context, host, columns_to_join=None):
                                     manual_joins=columns_to_join)
 
 
-def _instance_get_all_uuids_by_host(context, host):
-    """Return a list of the instance uuids on a given host.
+def _instance_get_all_uuids_by_hosts(context, hosts):
+    """Return a dict, keyed by hostname, of a list of the instance uuids on the
+    host for each supplied hostname.
 
-    Returns a list of UUIDs, not Instance model objects.
+    Returns a dict, keyed by hostname, of a list of UUIDs, not Instance model
+    objects.
     """
-    uuids = []
-    for tuple in model_query(context, models.Instance, (models.Instance.uuid,),
-                             read_deleted="no").\
-                filter_by(host=host).\
-                all():
-        uuids.append(tuple[0])
-    return uuids
+    itbl = models.Instance.__table__
+    default_deleted_value = itbl.c.deleted.default.arg
+    sel = sql.select([itbl.c.host, itbl.c.uuid])
+    sel = sel.where(sql.and_(
+            itbl.c.deleted == default_deleted_value,
+            itbl.c.host.in_(hosts)))
+
+    # group the instance UUIDs by hostname
+    res = collections.defaultdict(list)
+    for rec in context.session.execute(sel).fetchall():
+        res[rec[0]].append(rec[1])
+    return res
 
 
 @pick_context_manager_reader
-def instance_get_all_uuids_by_host(context, host):
-    return _instance_get_all_uuids_by_host(context, host)
+def instance_get_all_uuids_by_hosts(context, hosts):
+    return _instance_get_all_uuids_by_hosts(context, hosts)
 
 
 @pick_context_manager_reader

nova/objects/instance.py

@@ -1225,7 +1225,8 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
     # Version 2.3: Add get_count_by_vm_state()
     # Version 2.4: Add get_counts()
     # Version 2.5: Add get_uuids_by_host_and_node()
-    VERSION = '2.5'
+    # Version 2.6: Add get_uuids_by_hosts()
+    VERSION = '2.6'
 
     fields = {
         'objects': fields.ListOfObjectsField('Instance'),
@@ -1429,7 +1430,15 @@ class InstanceList(base.ObjectListBase, base.NovaObject):
 
     @base.remotable_classmethod
     def get_uuids_by_host(cls, context, host):
-        return db.instance_get_all_uuids_by_host(context, host)
+        return db.instance_get_all_uuids_by_hosts(context, [host]).get(
+            host, [])
+
+    @base.remotable_classmethod
+    def get_uuids_by_hosts(cls, context, hosts):
+        """Returns a dict, keyed by hypervisor hostname, of a list of instance
+        UUIDs associated with that compute node.
+        """
+        return db.instance_get_all_uuids_by_hosts(context, hosts)
 
     @staticmethod
     @db_api.pick_context_manager_reader

nova/tests/unit/db/test_db_api.py

@@ -357,13 +357,16 @@ class UnsupportedDbRegexpTestCase(DbTestCase):
                           self.context, {'display_name': '%test%'},
                           marker=uuidsentinel.uuid1)
 
-    def test_instance_get_all_uuids_by_host(self, mock_get_regexp):
+    def test_instance_get_all_uuids_by_hosts(self, mock_get_regexp):
         test1 = self.create_instance_with_args(display_name='test1')
         test2 = self.create_instance_with_args(display_name='test2')
         test3 = self.create_instance_with_args(display_name='test3')
         uuids = [i.uuid for i in (test1, test2, test3)]
-        found_uuids = db.instance_get_all_uuids_by_host(self.context,
-                                                        test1.host)
+        results = db.instance_get_all_uuids_by_hosts(self.context,
+                                                         [test1.host])
+        self.assertEqual(1, len(results))
+        self.assertIn(test1.host, results)
+        found_uuids = results[test1.host]
         self.assertEqual(sorted(uuids), sorted(found_uuids))
 
     def _assert_equals_inst_order(self, correct_order, filters,
@@ -821,21 +824,33 @@ class SqlAlchemyDbApiTestCase(DbTestCase):
         self.assertNotIn('info_cache', instance)
         self.assertNotIn('security_groups', instance)
 
-    def test_instance_get_all_uuids_by_host(self):
+    def test_instance_get_all_uuids_by_hosts(self):
         ctxt = context.get_admin_context()
         self.create_instance_with_args()
         self.create_instance_with_args()
         self.create_instance_with_args(host='host2')
 
         @sqlalchemy_api.pick_context_manager_reader
-        def test(context):
-            return sqlalchemy_api._instance_get_all_uuids_by_host(
-                context, 'host1')
+        def test1(context):
+            return sqlalchemy_api._instance_get_all_uuids_by_hosts(
+                context, ['host1'])
 
-        result = test(ctxt)
+        @sqlalchemy_api.pick_context_manager_reader
+        def test2(context):
+            return sqlalchemy_api._instance_get_all_uuids_by_hosts(
+                context, ['host1', 'host2'])
+
+        result = test1(ctxt)
+
+        self.assertEqual(1, len(result))
+        self.assertEqual(2, len(result['host1']))
+        self.assertEqual(six.text_type, type(result['host1'][0]))
+
+        result = test2(ctxt)
 
         self.assertEqual(2, len(result))
-        self.assertEqual(six.text_type, type(result[0]))
+        self.assertEqual(2, len(result['host1']))
+        self.assertEqual(1, len(result['host2']))
 
     @mock.patch('oslo_utils.uuidutils.generate_uuid')
     def test_instance_get_active_by_window_joined_paging(self, mock_uuids):

nova/tests/unit/objects/test_instance.py

@@ -1915,14 +1915,38 @@ class _TestInstanceListObject(object):
         self.assertEqual(2, len(instances))
         self.assertEqual([1, 2], [x.id for x in instances])
 
-    @mock.patch('nova.db.api.instance_get_all_uuids_by_host')
+    @mock.patch('nova.db.api.instance_get_all_uuids_by_hosts')
+    def test_get_uuids_by_host_no_match(self, mock_get_all):
+        mock_get_all.return_value = {}
+        actual_uuids = objects.InstanceList.get_uuids_by_host(
+            self.context, 'b')
+        self.assertEqual([], actual_uuids)
+        mock_get_all.assert_called_once_with(self.context, ['b'])
+
+    @mock.patch('nova.db.api.instance_get_all_uuids_by_hosts')
     def test_get_uuids_by_host(self, mock_get_all):
         fake_instances = [uuids.inst1, uuids.inst2]
-        mock_get_all.return_value = fake_instances
+        mock_get_all.return_value = {
+            'b': fake_instances
+        }
         actual_uuids = objects.InstanceList.get_uuids_by_host(
             self.context, 'b')
         self.assertEqual(fake_instances, actual_uuids)
-        mock_get_all.assert_called_once_with(self.context, 'b')
+        mock_get_all.assert_called_once_with(self.context, ['b'])
+
+    @mock.patch('nova.db.api.instance_get_all_uuids_by_hosts')
+    def test_get_uuids_by_hosts(self, mock_get_all):
+        fake_instances_a = [uuids.inst1, uuids.inst2]
+        fake_instances_b = [uuids.inst3, uuids.inst4]
+        fake_instances = {
+            'a': fake_instances_a,
+            'b': fake_instances_b
+        }
+        mock_get_all.return_value = fake_instances
+        actual_uuids = objects.InstanceList.get_uuids_by_hosts(
+            self.context, ['a', 'b'])
+        self.assertEqual(fake_instances, actual_uuids)
+        mock_get_all.assert_called_once_with(self.context, ['a', 'b'])
 
 
 class TestInstanceListObject(test_objects._LocalTest,

nova/tests/unit/objects/test_objects.py

@@ -1082,7 +1082,7 @@ object_data = {
     'InstanceGroup': '1.11-852ac511d30913ee88f3c3a869a8f30a',
     'InstanceGroupList': '1.8-90f8f1a445552bb3bbc9fa1ae7da27d4',
     'InstanceInfoCache': '1.5-cd8b96fefe0fc8d4d337243ba0bf0e1e',
-    'InstanceList': '2.5-43b5e7d8c4ebde52f2c2da22a1739e95',
+    'InstanceList': '2.6-238f125650c25d6d12722340d726f723',
     'InstanceMapping': '1.2-3bd375e65c8eb9c45498d2f87b882e03',
     'InstanceMappingList': '1.3-d34b6ebb076d542ae0f8b440534118da',
     'InstanceNUMACell': '1.4-7c1eb9a198dee076b4de0840e45f4f55',