Fix bugs in resource tracker and cleanup

Fixes bugs in resource tracker:
* Handle disk oversubscription
* Handle suspended/powered off instances

The usage model is changed to the old style that is
based on actual instance usage on a compute host.
(Not the current point in time of the hypervisor's
 reported host stats)

There is now a 'limits' filter property that can be passed from
the scheduler to the compute node to indicate that
oversubscription of resources is desired:

The 'limits' filter property is a dict with the following possible
keys:

* memory_mb - Specifies the memory ceiling for the compute node.
* disk_gb - Specifies the disk space ceiling for the compute node.
* vcpu - Specifies the max number of vcpus for the compute node.

There is also some general cleanup and additional unit tests in
an attempt to simplify down this function.

bug 1048842
bug 1052157

Change-Id: I6ee851b8c03234a78a64d9f5c494dfc7059cdda4
This commit is contained in:
Brian Elliott 2012-09-14 15:17:07 +00:00 committed by Chris Behrens
parent be99338950
commit a187d8885e
9 changed files with 158 additions and 50 deletions

View File

@ -132,6 +132,9 @@ class FilterScheduler(driver.Scheduler):
# Add a retry entry for the selected compute host: # Add a retry entry for the selected compute host:
self._add_retry_host(filter_properties, weighted_host.host_state.host) self._add_retry_host(filter_properties, weighted_host.host_state.host)
self._add_oversubscription_policy(filter_properties,
weighted_host.host_state)
payload = dict(request_spec=request_spec, payload = dict(request_spec=request_spec,
weighted_host=weighted_host.to_dict(), weighted_host=weighted_host.to_dict(),
instance_id=instance_uuid) instance_id=instance_uuid)
@ -160,6 +163,9 @@ class FilterScheduler(driver.Scheduler):
hosts = retry['hosts'] hosts = retry['hosts']
hosts.append(host) hosts.append(host)
def _add_oversubscription_policy(self, filter_properties, host_state):
filter_properties['limits'] = host_state.limits
def _get_configuration_options(self): def _get_configuration_options(self):
"""Fetch options dictionary. Broken out for testing.""" """Fetch options dictionary. Broken out for testing."""
return self.options.get_configuration() return self.options.get_configuration()

View File

@ -47,4 +47,10 @@ class CoreFilter(filters.BaseHostFilter):
instance_vcpus = instance_type['vcpus'] instance_vcpus = instance_type['vcpus']
vcpus_total = host_state.vcpus_total * FLAGS.cpu_allocation_ratio vcpus_total = host_state.vcpus_total * FLAGS.cpu_allocation_ratio
# Only provide a VCPU limit to compute if the virt driver is reporting
# an accurate count of installed VCPUs. (XenServer driver does not)
if vcpus_total > 0:
host_state.limits['vcpu'] = vcpus_total
return (vcpus_total - host_state.vcpus_used) >= instance_vcpus return (vcpus_total - host_state.vcpus_used) >= instance_vcpus

View File

@ -0,0 +1,54 @@
# Copyright (c) 2012 OpenStack, LLC.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from nova import flags
from nova.openstack.common import cfg
from nova.openstack.common import log as logging
from nova.scheduler import filters
LOG = logging.getLogger(__name__)
disk_allocation_ratio_opt = cfg.FloatOpt("disk_allocation_ratio", default=1.0,
help="virtual disk to physical disk allocation ratio")
FLAGS = flags.FLAGS
FLAGS.register_opt(disk_allocation_ratio_opt)
class DiskFilter(filters.BaseHostFilter):
"""Disk Filter with over subscription flag"""
def host_passes(self, host_state, filter_properties):
"""Filter based on disk usage"""
instance_type = filter_properties.get('instance_type')
requested_disk = 1024 * (instance_type['root_gb'] +
instance_type['ephemeral_gb'])
free_disk_mb = host_state.free_disk_mb
total_usable_disk_mb = host_state.total_usable_disk_gb * 1024
disk_mb_limit = total_usable_disk_mb * FLAGS.disk_allocation_ratio
used_disk_mb = total_usable_disk_mb - free_disk_mb
usable_disk_mb = disk_mb_limit - used_disk_mb
if not usable_disk_mb >= requested_disk:
LOG.debug(_("%(host_state)s does not have %(requested_disk)s MB "
"usable disk, it only has %(usable_disk_mb)s MB usable "
"disk."), locals())
return False
disk_gb_limit = disk_mb_limit / 1024
host_state.limits['disk_gb'] = disk_gb_limit
return True

View File

@ -39,17 +39,15 @@ class RamFilter(filters.BaseHostFilter):
free_ram_mb = host_state.free_ram_mb free_ram_mb = host_state.free_ram_mb
total_usable_ram_mb = host_state.total_usable_ram_mb total_usable_ram_mb = host_state.total_usable_ram_mb
oversubscribed_ram_limit_mb = (total_usable_ram_mb * memory_mb_limit = total_usable_ram_mb * FLAGS.ram_allocation_ratio
FLAGS.ram_allocation_ratio)
used_ram_mb = total_usable_ram_mb - free_ram_mb used_ram_mb = total_usable_ram_mb - free_ram_mb
usable_ram = oversubscribed_ram_limit_mb - used_ram_mb usable_ram = memory_mb_limit - used_ram_mb
if not usable_ram >= requested_ram: if not usable_ram >= requested_ram:
LOG.debug(_("%(host_state)s does not have %(requested_ram)s MB " LOG.debug(_("%(host_state)s does not have %(requested_ram)s MB "
"usable ram, it only has %(usable_ram)s MB usable ram."), "usable ram, it only has %(usable_ram)s MB usable ram."),
locals()) locals())
return False return False
# save oversubscribe ram limit so the compute host can verify # save oversubscription limit for compute node to test against:
# memory availability on builds: host_state.limits['memory_mb'] = memory_mb_limit
filter_properties['memory_mb_limit'] = oversubscribed_ram_limit_mb
return True return True

View File

@ -27,14 +27,7 @@ from nova.openstack.common import log as logging
from nova.openstack.common import timeutils from nova.openstack.common import timeutils
from nova.scheduler import filters from nova.scheduler import filters
host_manager_opts = [ host_manager_opts = [
cfg.IntOpt('reserved_host_disk_mb',
default=0,
help='Amount of disk in MB to reserve for host/dom0'),
cfg.IntOpt('reserved_host_memory_mb',
default=512,
help='Amount of memory in MB to reserve for host/dom0'),
cfg.MultiStrOpt('scheduler_available_filters', cfg.MultiStrOpt('scheduler_available_filters',
default=['nova.scheduler.filters.standard_filters'], default=['nova.scheduler.filters.standard_filters'],
help='Filter classes available to the scheduler which may ' help='Filter classes available to the scheduler which may '
@ -112,32 +105,31 @@ class HostState(object):
self.service = ReadOnlyDict(service) self.service = ReadOnlyDict(service)
# Mutable available resources. # Mutable available resources.
# These will change as resources are virtually "consumed". # These will change as resources are virtually "consumed".
self.total_usable_disk_gb = 0
self.disk_mb_used = 0
self.free_ram_mb = 0 self.free_ram_mb = 0
self.free_disk_mb = 0 self.free_disk_mb = 0
self.vcpus_total = 0 self.vcpus_total = 0
self.vcpus_used = 0 self.vcpus_used = 0
# Resource oversubscription values for the compute host:
self.limits = {}
def update_from_compute_node(self, compute): def update_from_compute_node(self, compute):
"""Update information about a host from its compute_node info.""" """Update information about a host from its compute_node info."""
all_disk_mb = compute['local_gb'] * 1024
all_ram_mb = compute['memory_mb'] all_ram_mb = compute['memory_mb']
# Assume virtual size is all consumed by instances if use qcow2 disk. # Assume virtual size is all consumed by instances if use qcow2 disk.
least = compute.get('disk_available_least') least = compute.get('disk_available_least')
free_disk_mb = least if least is not None else compute['free_disk_gb'] free_disk_mb = least if least is not None else compute['free_disk_gb']
free_disk_mb *= 1024 free_disk_mb *= 1024
free_ram_mb = compute['free_ram_mb']
if FLAGS.reserved_host_disk_mb > 0: self.disk_mb_used = compute['local_gb_used'] * 1024
all_disk_mb -= FLAGS.reserved_host_disk_mb
free_disk_mb -= FLAGS.reserved_host_disk_mb
if FLAGS.reserved_host_memory_mb > 0:
all_ram_mb -= FLAGS.reserved_host_memory_mb
free_ram_mb -= FLAGS.reserved_host_memory_mb
#NOTE(jogo) free_ram_mb can be negative #NOTE(jogo) free_ram_mb can be negative
self.free_ram_mb = free_ram_mb self.free_ram_mb = compute['free_ram_mb']
self.total_usable_ram_mb = all_ram_mb self.total_usable_ram_mb = all_ram_mb
self.total_usable_disk_gb = compute['local_gb']
self.free_disk_mb = free_disk_mb self.free_disk_mb = free_disk_mb
self.vcpus_total = compute['vcpus'] self.vcpus_total = compute['vcpus']
self.vcpus_used = compute['vcpus_used'] self.vcpus_used = compute['vcpus_used']

View File

@ -28,16 +28,20 @@ from nova.scheduler import host_manager
COMPUTE_NODES = [ COMPUTE_NODES = [
dict(id=1, local_gb=1024, memory_mb=1024, vcpus=1, dict(id=1, local_gb=1024, memory_mb=1024, vcpus=1,
disk_available_least=512, free_ram_mb=512, vcpus_used=1, disk_available_least=512, free_ram_mb=512, vcpus_used=1,
free_disk_mb=512, service=dict(host='host1', disabled=False)), free_disk_mb=512, local_gb_used=0,
service=dict(host='host1', disabled=False)),
dict(id=2, local_gb=2048, memory_mb=2048, vcpus=2, dict(id=2, local_gb=2048, memory_mb=2048, vcpus=2,
disk_available_least=1024, free_ram_mb=1024, vcpus_used=2, disk_available_least=1024, free_ram_mb=1024, vcpus_used=2,
free_disk_mb=1024, service=dict(host='host2', disabled=True)), free_disk_mb=1024, local_gb_used=0,
service=dict(host='host2', disabled=True)),
dict(id=3, local_gb=4096, memory_mb=4096, vcpus=4, dict(id=3, local_gb=4096, memory_mb=4096, vcpus=4,
disk_available_least=3072, free_ram_mb=3072, vcpus_used=1, disk_available_least=3072, free_ram_mb=3072, vcpus_used=1,
free_disk_mb=3072, service=dict(host='host3', disabled=False)), free_disk_mb=3072, local_gb_used=0,
service=dict(host='host3', disabled=False)),
dict(id=4, local_gb=8192, memory_mb=8192, vcpus=8, dict(id=4, local_gb=8192, memory_mb=8192, vcpus=8,
disk_available_least=8192, free_ram_mb=8192, vcpus_used=0, disk_available_least=8192, free_ram_mb=8192, vcpus_used=0,
free_disk_mb=8192, service=dict(host='host4', disabled=False)), free_disk_mb=8192, local_gb_used=0,
service=dict(host='host4', disabled=False)),
# Broken entry # Broken entry
dict(id=5, local_gb=1024, memory_mb=1024, vcpus=1, service=None), dict(id=5, local_gb=1024, memory_mb=1024, vcpus=1, service=None),
] ]

View File

@ -216,7 +216,6 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
self.assertEqual(info['called'], 0) self.assertEqual(info['called'], 0)
def test_get_cost_functions(self): def test_get_cost_functions(self):
self.flags(reserved_host_memory_mb=128)
fixture = fakes.FakeFilterScheduler() fixture = fakes.FakeFilterScheduler()
fns = fixture.get_cost_functions() fns = fixture.get_cost_functions()
self.assertEquals(len(fns), 1) self.assertEquals(len(fns), 1)
@ -225,8 +224,9 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
hostinfo = host_manager.HostState('host', 'compute') hostinfo = host_manager.HostState('host', 'compute')
hostinfo.update_from_compute_node(dict(memory_mb=1000, hostinfo.update_from_compute_node(dict(memory_mb=1000,
local_gb=0, vcpus=1, disk_available_least=1000, local_gb=0, vcpus=1, disk_available_least=1000,
free_disk_mb=1000, free_ram_mb=1000, vcpus_used=0)) free_disk_mb=1000, free_ram_mb=872, vcpus_used=0,
self.assertEquals(1000 - 128, fn(hostinfo, {})) local_gb_used=0))
self.assertEquals(872, fn(hostinfo, {}))
def test_max_attempts(self): def test_max_attempts(self):
self.flags(scheduler_max_attempts=4) self.flags(scheduler_max_attempts=4)

View File

@ -514,6 +514,18 @@ class HostFiltersTestCase(test.TestCase):
'capabilities': capabilities, 'service': service}) 'capabilities': capabilities, 'service': service})
self.assertFalse(filt_cls.host_passes(host, filter_properties)) self.assertFalse(filt_cls.host_passes(host, filter_properties))
def test_ram_filter_passes(self):
self._stub_service_is_up(True)
filt_cls = self.class_map['RamFilter']()
self.flags(ram_allocation_ratio=1.0)
filter_properties = {'instance_type': {'memory_mb': 1024}}
capabilities = {'enabled': True}
service = {'disabled': False}
host = fakes.FakeHostState('host1', 'compute',
{'free_ram_mb': 1024, 'total_usable_ram_mb': 1024,
'capabilities': capabilities, 'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties))
def test_ram_filter_oversubscribe(self): def test_ram_filter_oversubscribe(self):
self._stub_service_is_up(True) self._stub_service_is_up(True)
filt_cls = self.class_map['RamFilter']() filt_cls = self.class_map['RamFilter']()
@ -525,24 +537,62 @@ class HostFiltersTestCase(test.TestCase):
{'free_ram_mb': -1024, 'total_usable_ram_mb': 2048, {'free_ram_mb': -1024, 'total_usable_ram_mb': 2048,
'capabilities': capabilities, 'service': service}) 'capabilities': capabilities, 'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties)) self.assertTrue(filt_cls.host_passes(host, filter_properties))
self.assertEqual(2048 * 2.0, host.limits['memory_mb'])
def test_ram_filter_sets_memory_limit(self): def test_disk_filter_passes(self):
"""Test that ram filter sets a filter_property denoting the memory
ceiling.
"""
self._stub_service_is_up(True) self._stub_service_is_up(True)
filt_cls = self.class_map['RamFilter']() filt_cls = self.class_map['DiskFilter']()
self.flags(ram_allocation_ratio=2.0) self.flags(disk_allocation_ratio=1.0)
filter_properties = {'instance_type': {'memory_mb': 1024}} filter_properties = {'instance_type': {'root_gb': 1,
'ephemeral_gb': 1}}
capabilities = {'enabled': True} capabilities = {'enabled': True}
service = {'disabled': False} service = {'disabled': False}
host = fakes.FakeHostState('host1', 'compute', host = fakes.FakeHostState('host1', 'compute',
{'free_ram_mb': -1024, 'total_usable_ram_mb': 2048, {'free_disk_mb': 11 * 1024, 'total_usable_disk_gb': 13,
'capabilities': capabilities, 'service': service}) 'capabilities': capabilities, 'service': service})
filt_cls.host_passes(host, filter_properties) self.assertTrue(filt_cls.host_passes(host, filter_properties))
self.assertEqual(host.total_usable_ram_mb * 2.0, def test_disk_filter_fails(self):
filter_properties['memory_mb_limit']) self._stub_service_is_up(True)
filt_cls = self.class_map['DiskFilter']()
self.flags(disk_allocation_ratio=1.0)
filter_properties = {'instance_type': {'root_gb': 2,
'ephemeral_gb': 1}}
capabilities = {'enabled': True}
service = {'disabled': False}
host = fakes.FakeHostState('host1', 'compute',
{'free_disk_mb': 11 * 1024, 'total_usable_disk_gb': 13,
'capabilities': capabilities, 'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties))
def test_disk_filter_oversubscribe(self):
self._stub_service_is_up(True)
filt_cls = self.class_map['DiskFilter']()
self.flags(disk_allocation_ratio=10.0)
filter_properties = {'instance_type': {'root_gb': 100,
'ephemeral_gb': 19}}
capabilities = {'enabled': True}
service = {'disabled': False}
# 1GB used... so 119GB allowed...
host = fakes.FakeHostState('host1', 'compute',
{'free_disk_mb': 11 * 1024, 'total_usable_disk_gb': 12,
'capabilities': capabilities, 'service': service})
self.assertTrue(filt_cls.host_passes(host, filter_properties))
self.assertEqual(12 * 10.0, host.limits['disk_gb'])
def test_disk_filter_oversubscribe_fail(self):
self._stub_service_is_up(True)
filt_cls = self.class_map['DiskFilter']()
self.flags(disk_allocation_ratio=10.0)
filter_properties = {'instance_type': {'root_gb': 100,
'ephemeral_gb': 20}}
capabilities = {'enabled': True}
service = {'disabled': False}
# 1GB used... so 119GB allowed...
host = fakes.FakeHostState('host1', 'compute',
{'free_disk_mb': 11 * 1024, 'total_usable_disk_gb': 12,
'capabilities': capabilities, 'service': service})
self.assertFalse(filt_cls.host_passes(host, filter_properties))
def test_compute_filter_fails_on_service_disabled(self): def test_compute_filter_fails_on_service_disabled(self):
self._stub_service_is_up(True) self._stub_service_is_up(True)

View File

@ -122,8 +122,6 @@ class HostManagerTestCase(test.TestCase):
self.assertDictMatch(service_states, expected) self.assertDictMatch(service_states, expected)
def test_get_all_host_states(self): def test_get_all_host_states(self):
self.flags(reserved_host_memory_mb=512,
reserved_host_disk_mb=1024)
context = 'fake_context' context = 'fake_context'
topic = 'compute' topic = 'compute'
@ -145,18 +143,18 @@ class HostManagerTestCase(test.TestCase):
host = compute_node['service']['host'] host = compute_node['service']['host']
self.assertEqual(host_states[host].service, self.assertEqual(host_states[host].service,
compute_node['service']) compute_node['service'])
self.assertEqual(host_states['host1'].free_ram_mb, 0) self.assertEqual(host_states['host1'].free_ram_mb, 512)
# 511GB # 511GB
self.assertEqual(host_states['host1'].free_disk_mb, 523264) self.assertEqual(host_states['host1'].free_disk_mb, 524288)
self.assertEqual(host_states['host2'].free_ram_mb, 512) self.assertEqual(host_states['host2'].free_ram_mb, 1024)
# 1023GB # 1023GB
self.assertEqual(host_states['host2'].free_disk_mb, 1047552) self.assertEqual(host_states['host2'].free_disk_mb, 1048576)
self.assertEqual(host_states['host3'].free_ram_mb, 2560) self.assertEqual(host_states['host3'].free_ram_mb, 3072)
# 3071GB # 3071GB
self.assertEqual(host_states['host3'].free_disk_mb, 3144704) self.assertEqual(host_states['host3'].free_disk_mb, 3145728)
self.assertEqual(host_states['host4'].free_ram_mb, 7680) self.assertEqual(host_states['host4'].free_ram_mb, 8192)
# 8191GB # 8191GB
self.assertEqual(host_states['host4'].free_disk_mb, 8387584) self.assertEqual(host_states['host4'].free_disk_mb, 8388608)
class HostStateTestCase(test.TestCase): class HostStateTestCase(test.TestCase):