Make compute_node_statistics() use new schema

The nova.db.sqlalchemy.api.compute_node_statistics() method is being
called directly from the os-hosts API extension. This patch fixes up the
SQLAlchemy-fu to properly aggregate returned results when some compute
nodes have been migrated to the new resource-providers database schema.

Co-Authored-By: Ed Leafe <ed@leafe.com>

Partially-implements: compute-node-inventory-newton

Change-Id: I137266e4fa152c3916cb5196fb15b12c18a19886

nova/db/sqlalchemy/api.py

@@ -781,7 +781,11 @@ def _compute_node_select(context, filters=None):
     if "hypervisor_hostname" in filters:
         hyp_hostname = filters["hypervisor_hostname"]
         select = select.where(cn_tbl.c.hypervisor_hostname == hyp_hostname)
+    return select
 
+
+def _compute_node_fetchall(context, filters=None):
+    select = _compute_node_select(context, filters)
     engine = get_engine(context)
     conn = engine.connect()
 
@@ -795,7 +799,7 @@ def _compute_node_select(context, filters=None):
 
 @pick_context_manager_reader
 def compute_node_get(context, compute_id):
-    results = _compute_node_select(context, {"compute_id": compute_id})
+    results = _compute_node_fetchall(context, {"compute_id": compute_id})
     if not results:
         raise exception.ComputeHostNotFound(host=compute_id)
     return results[0]
@@ -815,7 +819,7 @@ def compute_node_get_model(context, compute_id):
 
 @pick_context_manager_reader
 def compute_nodes_get_by_service_id(context, service_id):
-    results = _compute_node_select(context, {"service_id": service_id})
+    results = _compute_node_fetchall(context, {"service_id": service_id})
     if not results:
         raise exception.ServiceNotFound(service_id=service_id)
     return results
@@ -823,7 +827,7 @@ def compute_nodes_get_by_service_id(context, service_id):
 
 @pick_context_manager_reader
 def compute_node_get_by_host_and_nodename(context, host, nodename):
-    results = _compute_node_select(context,
+    results = _compute_node_fetchall(context,
             {"host": host, "hypervisor_hostname": nodename})
     if not results:
         raise exception.ComputeHostNotFound(host=host)
@@ -832,7 +836,7 @@ def compute_node_get_by_host_and_nodename(context, host, nodename):
 
 @pick_context_manager_reader_allow_async
 def compute_node_get_all_by_host(context, host):
-    results = _compute_node_select(context, {"host": host})
+    results = _compute_node_fetchall(context, {"host": host})
     if not results:
         raise exception.ComputeHostNotFound(host=host)
     return results
@@ -840,7 +844,7 @@ def compute_node_get_all_by_host(context, host):
 
 @pick_context_manager_reader
 def compute_node_get_all(context):
-    return _compute_node_select(context)
+    return _compute_node_fetchall(context)
 
 
 @pick_context_manager_reader
@@ -895,38 +899,115 @@ def compute_node_delete(context, compute_id):
 @pick_context_manager_reader
 def compute_node_statistics(context):
     """Compute statistics over all compute nodes."""
+    engine = get_engine(context)
+    services_tbl = models.Service.__table__
+
+    inner_sel = sa.alias(_compute_node_select(context), name='inner_sel')
 
     # TODO(sbauza): Remove the service_id filter in a later release
     # once we are sure that all compute nodes report the host field
-    _filter = or_(models.Service.host == models.ComputeNode.host,
+    j = sa.join(
-                  models.Service.id == models.ComputeNode.service_id)
+        inner_sel, services_tbl,
+        sql.and_(
+            sql.or_(
+                inner_sel.c.host == services_tbl.c.host,
+                inner_sel.c.service_id == services_tbl.c.id
+            ),
+            services_tbl.c.disabled == false(),
+            services_tbl.c.binary == 'nova-compute'
+        )
+    )
 
-    result = model_query(context,
+    # NOTE(jaypipes): This COALESCE() stuff is temporary while the data
-                         models.ComputeNode, (
+    # migration to the new resource providers inventories and allocations
-                             func.count(models.ComputeNode.id),
+    # tables is completed.
-                             func.sum(models.ComputeNode.vcpus),
+    agg_cols = [
-                             func.sum(models.ComputeNode.memory_mb),
+        func.count().label('count'),
-                             func.sum(models.ComputeNode.local_gb),
+        sql.func.sum(
-                             func.sum(models.ComputeNode.vcpus_used),
+            sql.func.coalesce(
-                             func.sum(models.ComputeNode.memory_mb_used),
+                inner_sel.c.inv_vcpus,
-                             func.sum(models.ComputeNode.local_gb_used),
+                inner_sel.c.vcpus
-                             func.sum(models.ComputeNode.free_ram_mb),
+            )
-                             func.sum(models.ComputeNode.free_disk_gb),
+        ).label('vcpus'),
-                             func.sum(models.ComputeNode.current_workload),
+        sql.func.sum(
-                             func.sum(models.ComputeNode.running_vms),
+            sql.func.coalesce(
-                             func.sum(models.ComputeNode.disk_available_least),
+                inner_sel.c.inv_memory_mb,
-                         ), read_deleted="no").\
+                inner_sel.c.memory_mb
-                         filter(models.Service.disabled == false()).\
+            )
-                         filter(models.Service.binary == "nova-compute").\
+        ).label('memory_mb'),
-                         filter(_filter).\
+        sql.func.sum(
-                         first()
+            sql.func.coalesce(
+                inner_sel.c.inv_local_gb,
+                inner_sel.c.local_gb
+            )
+        ).label('local_gb'),
+        sql.func.sum(
+            sql.func.coalesce(
+                inner_sel.c.inv_vcpus_used,
+                inner_sel.c.vcpus_used
+            )
+        ).label('vcpus_used'),
+        sql.func.sum(
+            sql.func.coalesce(
+                inner_sel.c.inv_memory_mb_used,
+                inner_sel.c.memory_mb_used
+            )
+        ).label('memory_mb_used'),
+        sql.func.sum(
+            sql.func.coalesce(
+                inner_sel.c.inv_local_gb_used,
+                inner_sel.c.local_gb_used
+            )
+        ).label('local_gb_used'),
+        # NOTE(jaypipes): This mess cannot be removed until the
+        # resource-providers-allocations blueprint is completed and all of the
+        # data migrations for BOTH inventory and allocations fields have been
+        # completed.
+        sql.func.sum(
+            # NOTE(jaypipes): free_ram_mb and free_disk_gb do NOT take
+            # allocation ratios for those resources into account but they DO
+            # take reserved memory and disk configuration option amounts into
+            # account. Awesomesauce.
+            sql.func.coalesce(
+                (inner_sel.c.inv_memory_mb - (
+                    inner_sel.c.inv_memory_mb_used +
+                    inner_sel.c.inv_memory_mb_reserved)
+                ),
+                inner_sel.c.free_ram_mb
+            )
+        ).label('free_ram_mb'),
+        sql.func.sum(
+            sql.func.coalesce(
+                (inner_sel.c.inv_local_gb - (
+                    inner_sel.c.inv_local_gb_used +
+                    inner_sel.c.inv_local_gb_reserved)
+                ),
+                inner_sel.c.free_disk_gb
+            )
+        ).label('free_disk_gb'),
+        sql.func.sum(
+            inner_sel.c.current_workload
+        ).label('current_workload'),
+        sql.func.sum(
+            inner_sel.c.running_vms
+        ).label('running_vms'),
+        sql.func.sum(
+            inner_sel.c.disk_available_least
+        ).label('disk_available_least'),
+    ]
+    select = sql.select(agg_cols).select_from(j)
+    conn = engine.connect()
+
+    results = conn.execute(select).fetchone()
 
     # Build a dict of the info--making no assumptions about result
     fields = ('count', 'vcpus', 'memory_mb', 'local_gb', 'vcpus_used',
               'memory_mb_used', 'local_gb_used', 'free_ram_mb', 'free_disk_gb',
               'current_workload', 'running_vms', 'disk_available_least')
-    return {field: int(result[idx] or 0)
+    results = {field: int(results[idx] or 0)
-            for idx, field in enumerate(fields)}
+               for idx, field in enumerate(fields)}
+    conn.close()
+    return results
 
 
 ###################

nova/tests/unit/db/test_db_api.py

@@ -7425,7 +7425,7 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin):
         # entries under the new resource-providers schema return non-None
         # values for the inv_* fields in the returned list of dicts from
         # _compute_node_select().
-        nodes = sqlalchemy_api._compute_node_select(self.ctxt)
+        nodes = sqlalchemy_api._compute_node_fetchall(self.ctxt)
         self.assertEqual(1, len(nodes))
         node = nodes[0]
         self.assertIsNone(node['inv_memory_mb'])
@@ -7515,12 +7515,6 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin):
         self.assertEqual(16, node['inv_vcpus'])
         self.assertEqual(2, node['inv_vcpus_used'])
 
-    def test_compute_node_exec(self):
-        results = sqlalchemy_api._compute_node_select(self.ctxt)
-        self.assertIsInstance(results, list)
-        self.assertEqual(1, len(results))
-        self.assertIsInstance(results[0], dict)
-
     def test_compute_node_get_all_deleted_compute_node(self):
         # Create a service and compute node and ensure we can find its stats;
         # delete the service and compute node when done and loop again
@@ -7724,11 +7718,83 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin):
         self._assertEqualListsOfObjects(nodes_created, nodes,
                         ignored_keys=self._ignored_keys + ['stats', 'service'])
 
-    def test_compute_node_statistics(self):
+    def test_compute_node_statistics_no_resource_providers(self):
+        service_dict = dict(host='hostA', binary='nova-compute',
+                            topic=CONF.compute_topic, report_count=1,
+                            disabled=False)
+        service = db.service_create(self.ctxt, service_dict)
+        # Define the various values for the new compute node
+        new_vcpus = 4
+        new_memory_mb = 4096
+        new_local_gb = 2048
+        new_vcpus_used = 1
+        new_memory_mb_used = 1024
+        new_local_gb_used = 100
+        new_free_ram_mb = 3072
+        new_free_disk_gb = 1948
+        new_running_vms = 1
+        new_current_workload = 0
+
+        # Calculate the expected values by adding the values for the new
+        # compute node to those for self.item
+        itm = self.item
+        exp_count = 2
+        exp_vcpus = new_vcpus + itm['vcpus']
+        exp_memory_mb = new_memory_mb + itm['memory_mb']
+        exp_local_gb = new_local_gb + itm['local_gb']
+        exp_vcpus_used = new_vcpus_used + itm['vcpus_used']
+        exp_memory_mb_used = new_memory_mb_used + itm['memory_mb_used']
+        exp_local_gb_used = new_local_gb_used + itm['local_gb_used']
+        exp_free_ram_mb = new_free_ram_mb + itm['free_ram_mb']
+        exp_free_disk_gb = new_free_disk_gb + itm['free_disk_gb']
+        exp_running_vms = new_running_vms + itm['running_vms']
+        exp_current_workload = new_current_workload + itm['current_workload']
+
+        # Create the new compute node
+        compute_node_dict = dict(vcpus=new_vcpus,
+                                 memory_mb=new_memory_mb,
+                                 local_gb=new_local_gb,
+                                 uuid=uuidsentinel.fake_compute_node,
+                                 vcpus_used=new_vcpus_used,
+                                 memory_mb_used=new_memory_mb_used,
+                                 local_gb_used=new_local_gb_used,
+                                 free_ram_mb=new_free_ram_mb,
+                                 free_disk_gb=new_free_disk_gb,
+                                 hypervisor_type="xen",
+                                 hypervisor_version=1,
+                                 cpu_info="",
+                                 running_vms=new_running_vms,
+                                 current_workload=new_current_workload,
+                                 service_id=service['id'],
+                                 host=service['host'],
+                                 disk_available_least=100,
+                                 hypervisor_hostname='abracadabra',
+                                 host_ip='127.0.0.2',
+                                 supported_instances='',
+                                 pci_stats='',
+                                 metrics='',
+                                 extra_resources='',
+                                 cpu_allocation_ratio=16.0,
+                                 ram_allocation_ratio=1.5,
+                                 disk_allocation_ratio=1.0,
+                                 stats='',
+                                 numa_topology='')
+        db.compute_node_create(self.ctxt, compute_node_dict)
+
+        # Get the stats, and make sure the stats agree with the expected
+        # amounts.
         stats = db.compute_node_statistics(self.ctxt)
-        self.assertEqual(stats.pop('count'), 1)
+        self.assertEqual(exp_count, stats['count'])
-        for k, v in stats.items():
+        self.assertEqual(exp_vcpus, stats['vcpus'])
-            self.assertEqual(v, self.item[k])
+        self.assertEqual(exp_memory_mb, stats['memory_mb'])
+        self.assertEqual(exp_local_gb, stats['local_gb'])
+        self.assertEqual(exp_vcpus_used, stats['vcpus_used'])
+        self.assertEqual(exp_memory_mb_used, stats['memory_mb_used'])
+        self.assertEqual(exp_local_gb_used, stats['local_gb_used'])
+        self.assertEqual(exp_free_ram_mb, stats['free_ram_mb'])
+        self.assertEqual(exp_free_disk_gb, stats['free_disk_gb'])
+        self.assertEqual(exp_running_vms, stats['running_vms'])
+        self.assertEqual(exp_current_workload, stats['current_workload'])
 
     def test_compute_node_statistics_disabled_service(self):
         serv = db.service_get_by_host_and_topic(