Make compute_node_statistics() use new schema
The nova.db.sqlalchemy.api.compute_node_statistics() method is being called directly from the os-hosts API extension. This patch fixes up the SQLAlchemy-fu to properly aggregate returned results when some compute nodes have been migrated to the new resource-providers database schema. Co-Authored-By: Ed Leafe <ed@leafe.com> Partially-implements: compute-node-inventory-newton Change-Id: I137266e4fa152c3916cb5196fb15b12c18a19886
This commit is contained in:
Jay Pipes
@@ -781,7 +781,11 @@ def _compute_node_select(context, filters=None):
|
||||
if "hypervisor_hostname" in filters:
|
||||
hyp_hostname = filters["hypervisor_hostname"]
|
||||
select = select.where(cn_tbl.c.hypervisor_hostname == hyp_hostname)
|
||||
return select
|
||||
|
||||
|
||||
def _compute_node_fetchall(context, filters=None):
|
||||
select = _compute_node_select(context, filters)
|
||||
engine = get_engine(context)
|
||||
conn = engine.connect()
|
||||
|
||||
@@ -795,7 +799,7 @@ def _compute_node_select(context, filters=None):
|
||||
|
||||
@pick_context_manager_reader
|
||||
def compute_node_get(context, compute_id):
|
||||
results = _compute_node_select(context, {"compute_id": compute_id})
|
||||
results = _compute_node_fetchall(context, {"compute_id": compute_id})
|
||||
if not results:
|
||||
raise exception.ComputeHostNotFound(host=compute_id)
|
||||
return results[0]
|
||||
@@ -815,7 +819,7 @@ def compute_node_get_model(context, compute_id):
|
||||
|
||||
@pick_context_manager_reader
|
||||
def compute_nodes_get_by_service_id(context, service_id):
|
||||
results = _compute_node_select(context, {"service_id": service_id})
|
||||
results = _compute_node_fetchall(context, {"service_id": service_id})
|
||||
if not results:
|
||||
raise exception.ServiceNotFound(service_id=service_id)
|
||||
return results
|
||||
@@ -823,7 +827,7 @@ def compute_nodes_get_by_service_id(context, service_id):
|
||||
|
||||
@pick_context_manager_reader
|
||||
def compute_node_get_by_host_and_nodename(context, host, nodename):
|
||||
results = _compute_node_select(context,
|
||||
results = _compute_node_fetchall(context,
|
||||
{"host": host, "hypervisor_hostname": nodename})
|
||||
if not results:
|
||||
raise exception.ComputeHostNotFound(host=host)
|
||||
@@ -832,7 +836,7 @@ def compute_node_get_by_host_and_nodename(context, host, nodename):
|
||||
|
||||
@pick_context_manager_reader_allow_async
|
||||
def compute_node_get_all_by_host(context, host):
|
||||
results = _compute_node_select(context, {"host": host})
|
||||
results = _compute_node_fetchall(context, {"host": host})
|
||||
if not results:
|
||||
raise exception.ComputeHostNotFound(host=host)
|
||||
return results
|
||||
@@ -840,7 +844,7 @@ def compute_node_get_all_by_host(context, host):
|
||||
|
||||
@pick_context_manager_reader
|
||||
def compute_node_get_all(context):
|
||||
return _compute_node_select(context)
|
||||
return _compute_node_fetchall(context)
|
||||
|
||||
|
||||
@pick_context_manager_reader
|
||||
@@ -895,38 +899,115 @@ def compute_node_delete(context, compute_id):
|
||||
@pick_context_manager_reader
|
||||
def compute_node_statistics(context):
|
||||
"""Compute statistics over all compute nodes."""
|
||||
engine = get_engine(context)
|
||||
services_tbl = models.Service.__table__
|
||||
|
||||
inner_sel = sa.alias(_compute_node_select(context), name='inner_sel')
|
||||
|
||||
# TODO(sbauza): Remove the service_id filter in a later release
|
||||
# once we are sure that all compute nodes report the host field
|
||||
_filter = or_(models.Service.host == models.ComputeNode.host,
|
||||
models.Service.id == models.ComputeNode.service_id)
|
||||
j = sa.join(
|
||||
inner_sel, services_tbl,
|
||||
sql.and_(
|
||||
sql.or_(
|
||||
inner_sel.c.host == services_tbl.c.host,
|
||||
inner_sel.c.service_id == services_tbl.c.id
|
||||
),
|
||||
services_tbl.c.disabled == false(),
|
||||
services_tbl.c.binary == 'nova-compute'
|
||||
)
|
||||
)
|
||||
|
||||
result = model_query(context,
|
||||
models.ComputeNode, (
|
||||
func.count(models.ComputeNode.id),
|
||||
func.sum(models.ComputeNode.vcpus),
|
||||
func.sum(models.ComputeNode.memory_mb),
|
||||
func.sum(models.ComputeNode.local_gb),
|
||||
func.sum(models.ComputeNode.vcpus_used),
|
||||
func.sum(models.ComputeNode.memory_mb_used),
|
||||
func.sum(models.ComputeNode.local_gb_used),
|
||||
func.sum(models.ComputeNode.free_ram_mb),
|
||||
func.sum(models.ComputeNode.free_disk_gb),
|
||||
func.sum(models.ComputeNode.current_workload),
|
||||
func.sum(models.ComputeNode.running_vms),
|
||||
func.sum(models.ComputeNode.disk_available_least),
|
||||
), read_deleted="no").\
|
||||
filter(models.Service.disabled == false()).\
|
||||
filter(models.Service.binary == "nova-compute").\
|
||||
filter(_filter).\
|
||||
first()
|
||||
# NOTE(jaypipes): This COALESCE() stuff is temporary while the data
|
||||
# migration to the new resource providers inventories and allocations
|
||||
# tables is completed.
|
||||
agg_cols = [
|
||||
func.count().label('count'),
|
||||
sql.func.sum(
|
||||
sql.func.coalesce(
|
||||
inner_sel.c.inv_vcpus,
|
||||
inner_sel.c.vcpus
|
||||
)
|
||||
).label('vcpus'),
|
||||
sql.func.sum(
|
||||
sql.func.coalesce(
|
||||
inner_sel.c.inv_memory_mb,
|
||||
inner_sel.c.memory_mb
|
||||
)
|
||||
).label('memory_mb'),
|
||||
sql.func.sum(
|
||||
sql.func.coalesce(
|
||||
inner_sel.c.inv_local_gb,
|
||||
inner_sel.c.local_gb
|
||||
)
|
||||
).label('local_gb'),
|
||||
sql.func.sum(
|
||||
sql.func.coalesce(
|
||||
inner_sel.c.inv_vcpus_used,
|
||||
inner_sel.c.vcpus_used
|
||||
)
|
||||
).label('vcpus_used'),
|
||||
sql.func.sum(
|
||||
sql.func.coalesce(
|
||||
inner_sel.c.inv_memory_mb_used,
|
||||
inner_sel.c.memory_mb_used
|
||||
)
|
||||
).label('memory_mb_used'),
|
||||
sql.func.sum(
|
||||
sql.func.coalesce(
|
||||
inner_sel.c.inv_local_gb_used,
|
||||
inner_sel.c.local_gb_used
|
||||
)
|
||||
).label('local_gb_used'),
|
||||
# NOTE(jaypipes): This mess cannot be removed until the
|
||||
# resource-providers-allocations blueprint is completed and all of the
|
||||
# data migrations for BOTH inventory and allocations fields have been
|
||||
# completed.
|
||||
sql.func.sum(
|
||||
# NOTE(jaypipes): free_ram_mb and free_disk_gb do NOT take
|
||||
# allocation ratios for those resources into account but they DO
|
||||
# take reserved memory and disk configuration option amounts into
|
||||
# account. Awesomesauce.
|
||||
sql.func.coalesce(
|
||||
(inner_sel.c.inv_memory_mb - (
|
||||
inner_sel.c.inv_memory_mb_used +
|
||||
inner_sel.c.inv_memory_mb_reserved)
|
||||
),
|
||||
inner_sel.c.free_ram_mb
|
||||
)
|
||||
).label('free_ram_mb'),
|
||||
sql.func.sum(
|
||||
sql.func.coalesce(
|
||||
(inner_sel.c.inv_local_gb - (
|
||||
inner_sel.c.inv_local_gb_used +
|
||||
inner_sel.c.inv_local_gb_reserved)
|
||||
),
|
||||
inner_sel.c.free_disk_gb
|
||||
)
|
||||
).label('free_disk_gb'),
|
||||
sql.func.sum(
|
||||
inner_sel.c.current_workload
|
||||
).label('current_workload'),
|
||||
sql.func.sum(
|
||||
inner_sel.c.running_vms
|
||||
).label('running_vms'),
|
||||
sql.func.sum(
|
||||
inner_sel.c.disk_available_least
|
||||
).label('disk_available_least'),
|
||||
]
|
||||
select = sql.select(agg_cols).select_from(j)
|
||||
conn = engine.connect()
|
||||
|
||||
results = conn.execute(select).fetchone()
|
||||
|
||||
# Build a dict of the info--making no assumptions about result
|
||||
fields = ('count', 'vcpus', 'memory_mb', 'local_gb', 'vcpus_used',
|
||||
'memory_mb_used', 'local_gb_used', 'free_ram_mb', 'free_disk_gb',
|
||||
'current_workload', 'running_vms', 'disk_available_least')
|
||||
return {field: int(result[idx] or 0)
|
||||
results = {field: int(results[idx] or 0)
|
||||
for idx, field in enumerate(fields)}
|
||||
conn.close()
|
||||
return results
|
||||
|
||||
|
||||
###################
|
||||
|
||||
@@ -7425,7 +7425,7 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin):
|
||||
# entries under the new resource-providers schema return non-None
|
||||
# values for the inv_* fields in the returned list of dicts from
|
||||
# _compute_node_select().
|
||||
nodes = sqlalchemy_api._compute_node_select(self.ctxt)
|
||||
nodes = sqlalchemy_api._compute_node_fetchall(self.ctxt)
|
||||
self.assertEqual(1, len(nodes))
|
||||
node = nodes[0]
|
||||
self.assertIsNone(node['inv_memory_mb'])
|
||||
@@ -7515,12 +7515,6 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin):
|
||||
self.assertEqual(16, node['inv_vcpus'])
|
||||
self.assertEqual(2, node['inv_vcpus_used'])
|
||||
|
||||
def test_compute_node_exec(self):
|
||||
results = sqlalchemy_api._compute_node_select(self.ctxt)
|
||||
self.assertIsInstance(results, list)
|
||||
self.assertEqual(1, len(results))
|
||||
self.assertIsInstance(results[0], dict)
|
||||
|
||||
def test_compute_node_get_all_deleted_compute_node(self):
|
||||
# Create a service and compute node and ensure we can find its stats;
|
||||
# delete the service and compute node when done and loop again
|
||||
@@ -7724,11 +7718,83 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin):
|
||||
self._assertEqualListsOfObjects(nodes_created, nodes,
|
||||
ignored_keys=self._ignored_keys + ['stats', 'service'])
|
||||
|
||||
def test_compute_node_statistics(self):
|
||||
def test_compute_node_statistics_no_resource_providers(self):
|
||||
service_dict = dict(host='hostA', binary='nova-compute',
|
||||
topic=CONF.compute_topic, report_count=1,
|
||||
disabled=False)
|
||||
service = db.service_create(self.ctxt, service_dict)
|
||||
# Define the various values for the new compute node
|
||||
new_vcpus = 4
|
||||
new_memory_mb = 4096
|
||||
new_local_gb = 2048
|
||||
new_vcpus_used = 1
|
||||
new_memory_mb_used = 1024
|
||||
new_local_gb_used = 100
|
||||
new_free_ram_mb = 3072
|
||||
new_free_disk_gb = 1948
|
||||
new_running_vms = 1
|
||||
new_current_workload = 0
|
||||
|
||||
# Calculate the expected values by adding the values for the new
|
||||
# compute node to those for self.item
|
||||
itm = self.item
|
||||
exp_count = 2
|
||||
exp_vcpus = new_vcpus + itm['vcpus']
|
||||
exp_memory_mb = new_memory_mb + itm['memory_mb']
|
||||
exp_local_gb = new_local_gb + itm['local_gb']
|
||||
exp_vcpus_used = new_vcpus_used + itm['vcpus_used']
|
||||
exp_memory_mb_used = new_memory_mb_used + itm['memory_mb_used']
|
||||
exp_local_gb_used = new_local_gb_used + itm['local_gb_used']
|
||||
exp_free_ram_mb = new_free_ram_mb + itm['free_ram_mb']
|
||||
exp_free_disk_gb = new_free_disk_gb + itm['free_disk_gb']
|
||||
exp_running_vms = new_running_vms + itm['running_vms']
|
||||
exp_current_workload = new_current_workload + itm['current_workload']
|
||||
|
||||
# Create the new compute node
|
||||
compute_node_dict = dict(vcpus=new_vcpus,
|
||||
memory_mb=new_memory_mb,
|
||||
local_gb=new_local_gb,
|
||||
uuid=uuidsentinel.fake_compute_node,
|
||||
vcpus_used=new_vcpus_used,
|
||||
memory_mb_used=new_memory_mb_used,
|
||||
local_gb_used=new_local_gb_used,
|
||||
free_ram_mb=new_free_ram_mb,
|
||||
free_disk_gb=new_free_disk_gb,
|
||||
hypervisor_type="xen",
|
||||
hypervisor_version=1,
|
||||
cpu_info="",
|
||||
running_vms=new_running_vms,
|
||||
current_workload=new_current_workload,
|
||||
service_id=service['id'],
|
||||
host=service['host'],
|
||||
disk_available_least=100,
|
||||
hypervisor_hostname='abracadabra',
|
||||
host_ip='127.0.0.2',
|
||||
supported_instances='',
|
||||
pci_stats='',
|
||||
metrics='',
|
||||
extra_resources='',
|
||||
cpu_allocation_ratio=16.0,
|
||||
ram_allocation_ratio=1.5,
|
||||
disk_allocation_ratio=1.0,
|
||||
stats='',
|
||||
numa_topology='')
|
||||
db.compute_node_create(self.ctxt, compute_node_dict)
|
||||
|
||||
# Get the stats, and make sure the stats agree with the expected
|
||||
# amounts.
|
||||
stats = db.compute_node_statistics(self.ctxt)
|
||||
self.assertEqual(stats.pop('count'), 1)
|
||||
for k, v in stats.items():
|
||||
self.assertEqual(v, self.item[k])
|
||||
self.assertEqual(exp_count, stats['count'])
|
||||
self.assertEqual(exp_vcpus, stats['vcpus'])
|
||||
self.assertEqual(exp_memory_mb, stats['memory_mb'])
|
||||
self.assertEqual(exp_local_gb, stats['local_gb'])
|
||||
self.assertEqual(exp_vcpus_used, stats['vcpus_used'])
|
||||
self.assertEqual(exp_memory_mb_used, stats['memory_mb_used'])
|
||||
self.assertEqual(exp_local_gb_used, stats['local_gb_used'])
|
||||
self.assertEqual(exp_free_ram_mb, stats['free_ram_mb'])
|
||||
self.assertEqual(exp_free_disk_gb, stats['free_disk_gb'])
|
||||
self.assertEqual(exp_running_vms, stats['running_vms'])
|
||||
self.assertEqual(exp_current_workload, stats['current_workload'])
|
||||
|
||||
def test_compute_node_statistics_disabled_service(self):
|
||||
serv = db.service_get_by_host_and_topic(
|
||||
|
||||
Reference in New Issue
Block a user