diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 346ab360e75e..c250914a9870 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -781,7 +781,11 @@ def _compute_node_select(context, filters=None): if "hypervisor_hostname" in filters: hyp_hostname = filters["hypervisor_hostname"] select = select.where(cn_tbl.c.hypervisor_hostname == hyp_hostname) + return select + +def _compute_node_fetchall(context, filters=None): + select = _compute_node_select(context, filters) engine = get_engine(context) conn = engine.connect() @@ -795,7 +799,7 @@ def _compute_node_select(context, filters=None): @pick_context_manager_reader def compute_node_get(context, compute_id): - results = _compute_node_select(context, {"compute_id": compute_id}) + results = _compute_node_fetchall(context, {"compute_id": compute_id}) if not results: raise exception.ComputeHostNotFound(host=compute_id) return results[0] @@ -815,7 +819,7 @@ def compute_node_get_model(context, compute_id): @pick_context_manager_reader def compute_nodes_get_by_service_id(context, service_id): - results = _compute_node_select(context, {"service_id": service_id}) + results = _compute_node_fetchall(context, {"service_id": service_id}) if not results: raise exception.ServiceNotFound(service_id=service_id) return results @@ -823,7 +827,7 @@ def compute_nodes_get_by_service_id(context, service_id): @pick_context_manager_reader def compute_node_get_by_host_and_nodename(context, host, nodename): - results = _compute_node_select(context, + results = _compute_node_fetchall(context, {"host": host, "hypervisor_hostname": nodename}) if not results: raise exception.ComputeHostNotFound(host=host) @@ -832,7 +836,7 @@ def compute_node_get_by_host_and_nodename(context, host, nodename): @pick_context_manager_reader_allow_async def compute_node_get_all_by_host(context, host): - results = _compute_node_select(context, {"host": host}) + results = _compute_node_fetchall(context, {"host": host}) if not results: raise exception.ComputeHostNotFound(host=host) return results @@ -840,7 +844,7 @@ def compute_node_get_all_by_host(context, host): @pick_context_manager_reader def compute_node_get_all(context): - return _compute_node_select(context) + return _compute_node_fetchall(context) @pick_context_manager_reader @@ -895,38 +899,115 @@ def compute_node_delete(context, compute_id): @pick_context_manager_reader def compute_node_statistics(context): """Compute statistics over all compute nodes.""" + engine = get_engine(context) + services_tbl = models.Service.__table__ + + inner_sel = sa.alias(_compute_node_select(context), name='inner_sel') # TODO(sbauza): Remove the service_id filter in a later release # once we are sure that all compute nodes report the host field - _filter = or_(models.Service.host == models.ComputeNode.host, - models.Service.id == models.ComputeNode.service_id) + j = sa.join( + inner_sel, services_tbl, + sql.and_( + sql.or_( + inner_sel.c.host == services_tbl.c.host, + inner_sel.c.service_id == services_tbl.c.id + ), + services_tbl.c.disabled == false(), + services_tbl.c.binary == 'nova-compute' + ) + ) - result = model_query(context, - models.ComputeNode, ( - func.count(models.ComputeNode.id), - func.sum(models.ComputeNode.vcpus), - func.sum(models.ComputeNode.memory_mb), - func.sum(models.ComputeNode.local_gb), - func.sum(models.ComputeNode.vcpus_used), - func.sum(models.ComputeNode.memory_mb_used), - func.sum(models.ComputeNode.local_gb_used), - func.sum(models.ComputeNode.free_ram_mb), - func.sum(models.ComputeNode.free_disk_gb), - func.sum(models.ComputeNode.current_workload), - func.sum(models.ComputeNode.running_vms), - func.sum(models.ComputeNode.disk_available_least), - ), read_deleted="no").\ - filter(models.Service.disabled == false()).\ - filter(models.Service.binary == "nova-compute").\ - filter(_filter).\ - first() + # NOTE(jaypipes): This COALESCE() stuff is temporary while the data + # migration to the new resource providers inventories and allocations + # tables is completed. + agg_cols = [ + func.count().label('count'), + sql.func.sum( + sql.func.coalesce( + inner_sel.c.inv_vcpus, + inner_sel.c.vcpus + ) + ).label('vcpus'), + sql.func.sum( + sql.func.coalesce( + inner_sel.c.inv_memory_mb, + inner_sel.c.memory_mb + ) + ).label('memory_mb'), + sql.func.sum( + sql.func.coalesce( + inner_sel.c.inv_local_gb, + inner_sel.c.local_gb + ) + ).label('local_gb'), + sql.func.sum( + sql.func.coalesce( + inner_sel.c.inv_vcpus_used, + inner_sel.c.vcpus_used + ) + ).label('vcpus_used'), + sql.func.sum( + sql.func.coalesce( + inner_sel.c.inv_memory_mb_used, + inner_sel.c.memory_mb_used + ) + ).label('memory_mb_used'), + sql.func.sum( + sql.func.coalesce( + inner_sel.c.inv_local_gb_used, + inner_sel.c.local_gb_used + ) + ).label('local_gb_used'), + # NOTE(jaypipes): This mess cannot be removed until the + # resource-providers-allocations blueprint is completed and all of the + # data migrations for BOTH inventory and allocations fields have been + # completed. + sql.func.sum( + # NOTE(jaypipes): free_ram_mb and free_disk_gb do NOT take + # allocation ratios for those resources into account but they DO + # take reserved memory and disk configuration option amounts into + # account. Awesomesauce. + sql.func.coalesce( + (inner_sel.c.inv_memory_mb - ( + inner_sel.c.inv_memory_mb_used + + inner_sel.c.inv_memory_mb_reserved) + ), + inner_sel.c.free_ram_mb + ) + ).label('free_ram_mb'), + sql.func.sum( + sql.func.coalesce( + (inner_sel.c.inv_local_gb - ( + inner_sel.c.inv_local_gb_used + + inner_sel.c.inv_local_gb_reserved) + ), + inner_sel.c.free_disk_gb + ) + ).label('free_disk_gb'), + sql.func.sum( + inner_sel.c.current_workload + ).label('current_workload'), + sql.func.sum( + inner_sel.c.running_vms + ).label('running_vms'), + sql.func.sum( + inner_sel.c.disk_available_least + ).label('disk_available_least'), + ] + select = sql.select(agg_cols).select_from(j) + conn = engine.connect() + + results = conn.execute(select).fetchone() # Build a dict of the info--making no assumptions about result fields = ('count', 'vcpus', 'memory_mb', 'local_gb', 'vcpus_used', 'memory_mb_used', 'local_gb_used', 'free_ram_mb', 'free_disk_gb', 'current_workload', 'running_vms', 'disk_available_least') - return {field: int(result[idx] or 0) - for idx, field in enumerate(fields)} + results = {field: int(results[idx] or 0) + for idx, field in enumerate(fields)} + conn.close() + return results ################### diff --git a/nova/tests/unit/db/test_db_api.py b/nova/tests/unit/db/test_db_api.py index 933905d2bc16..f7ec94e90622 100644 --- a/nova/tests/unit/db/test_db_api.py +++ b/nova/tests/unit/db/test_db_api.py @@ -7425,7 +7425,7 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin): # entries under the new resource-providers schema return non-None # values for the inv_* fields in the returned list of dicts from # _compute_node_select(). - nodes = sqlalchemy_api._compute_node_select(self.ctxt) + nodes = sqlalchemy_api._compute_node_fetchall(self.ctxt) self.assertEqual(1, len(nodes)) node = nodes[0] self.assertIsNone(node['inv_memory_mb']) @@ -7515,12 +7515,6 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin): self.assertEqual(16, node['inv_vcpus']) self.assertEqual(2, node['inv_vcpus_used']) - def test_compute_node_exec(self): - results = sqlalchemy_api._compute_node_select(self.ctxt) - self.assertIsInstance(results, list) - self.assertEqual(1, len(results)) - self.assertIsInstance(results[0], dict) - def test_compute_node_get_all_deleted_compute_node(self): # Create a service and compute node and ensure we can find its stats; # delete the service and compute node when done and loop again @@ -7724,11 +7718,83 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin): self._assertEqualListsOfObjects(nodes_created, nodes, ignored_keys=self._ignored_keys + ['stats', 'service']) - def test_compute_node_statistics(self): + def test_compute_node_statistics_no_resource_providers(self): + service_dict = dict(host='hostA', binary='nova-compute', + topic=CONF.compute_topic, report_count=1, + disabled=False) + service = db.service_create(self.ctxt, service_dict) + # Define the various values for the new compute node + new_vcpus = 4 + new_memory_mb = 4096 + new_local_gb = 2048 + new_vcpus_used = 1 + new_memory_mb_used = 1024 + new_local_gb_used = 100 + new_free_ram_mb = 3072 + new_free_disk_gb = 1948 + new_running_vms = 1 + new_current_workload = 0 + + # Calculate the expected values by adding the values for the new + # compute node to those for self.item + itm = self.item + exp_count = 2 + exp_vcpus = new_vcpus + itm['vcpus'] + exp_memory_mb = new_memory_mb + itm['memory_mb'] + exp_local_gb = new_local_gb + itm['local_gb'] + exp_vcpus_used = new_vcpus_used + itm['vcpus_used'] + exp_memory_mb_used = new_memory_mb_used + itm['memory_mb_used'] + exp_local_gb_used = new_local_gb_used + itm['local_gb_used'] + exp_free_ram_mb = new_free_ram_mb + itm['free_ram_mb'] + exp_free_disk_gb = new_free_disk_gb + itm['free_disk_gb'] + exp_running_vms = new_running_vms + itm['running_vms'] + exp_current_workload = new_current_workload + itm['current_workload'] + + # Create the new compute node + compute_node_dict = dict(vcpus=new_vcpus, + memory_mb=new_memory_mb, + local_gb=new_local_gb, + uuid=uuidsentinel.fake_compute_node, + vcpus_used=new_vcpus_used, + memory_mb_used=new_memory_mb_used, + local_gb_used=new_local_gb_used, + free_ram_mb=new_free_ram_mb, + free_disk_gb=new_free_disk_gb, + hypervisor_type="xen", + hypervisor_version=1, + cpu_info="", + running_vms=new_running_vms, + current_workload=new_current_workload, + service_id=service['id'], + host=service['host'], + disk_available_least=100, + hypervisor_hostname='abracadabra', + host_ip='127.0.0.2', + supported_instances='', + pci_stats='', + metrics='', + extra_resources='', + cpu_allocation_ratio=16.0, + ram_allocation_ratio=1.5, + disk_allocation_ratio=1.0, + stats='', + numa_topology='') + db.compute_node_create(self.ctxt, compute_node_dict) + + # Get the stats, and make sure the stats agree with the expected + # amounts. stats = db.compute_node_statistics(self.ctxt) - self.assertEqual(stats.pop('count'), 1) - for k, v in stats.items(): - self.assertEqual(v, self.item[k]) + self.assertEqual(exp_count, stats['count']) + self.assertEqual(exp_vcpus, stats['vcpus']) + self.assertEqual(exp_memory_mb, stats['memory_mb']) + self.assertEqual(exp_local_gb, stats['local_gb']) + self.assertEqual(exp_vcpus_used, stats['vcpus_used']) + self.assertEqual(exp_memory_mb_used, stats['memory_mb_used']) + self.assertEqual(exp_local_gb_used, stats['local_gb_used']) + self.assertEqual(exp_free_ram_mb, stats['free_ram_mb']) + self.assertEqual(exp_free_disk_gb, stats['free_disk_gb']) + self.assertEqual(exp_running_vms, stats['running_vms']) + self.assertEqual(exp_current_workload, stats['current_workload']) def test_compute_node_statistics_disabled_service(self): serv = db.service_get_by_host_and_topic(