diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 2869b4fe13d5..eb115d8dc1a1 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -50,7 +50,6 @@ from nova import volume FLAGS = flags.FLAGS flags.DECLARE('dhcp_domain', 'nova.network.manager') -flags.DECLARE('service_down_time', 'nova.scheduler.driver') LOG = logging.getLogger("nova.api.ec2.cloud") @@ -290,8 +289,7 @@ class CloudController(object): hsvcs = [service for service in services \ if service['host'] == host] for svc in hsvcs: - delta = now - (svc['updated_at'] or svc['created_at']) - alive = (delta.seconds <= FLAGS.service_down_time) + alive = utils.service_is_up(svc) art = (alive and ":-)") or "XXX" active = 'enabled' if svc['disabled']: diff --git a/nova/flags.py b/nova/flags.py index cf02c9715a1e..f89e53796159 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -464,3 +464,6 @@ DEFINE_integer('zombie_instance_updated_at_window', 172800, 'being cleaned up.') DEFINE_boolean('allow_ec2_admin_api', False, 'Enable/Disable EC2 Admin API') + +DEFINE_integer('service_down_time', 60, + 'maximum time since last check-in for up service') diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 46d7046da01d..a0d718d5fa5a 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -37,8 +37,6 @@ from nova import utils FLAGS = flags.FLAGS LOG = logging.getLogger('nova.scheduler.driver') -flags.DEFINE_integer('service_down_time', 60, - 'maximum time since last check-in for up service') flags.DEFINE_string('scheduler_host_manager', 'nova.scheduler.host_manager.HostManager', 'The scheduler host manager class to use') @@ -159,21 +157,13 @@ class Scheduler(object): """Poll child zones periodically to get status.""" return self.zone_manager.update(context) - @staticmethod - def service_is_up(service): - """Check whether a service is up based on last heartbeat.""" - last_heartbeat = service['updated_at'] or service['created_at'] - # Timestamps in DB are UTC. - elapsed = utils.total_seconds(utils.utcnow() - last_heartbeat) - return abs(elapsed) <= FLAGS.service_down_time - def hosts_up(self, context, topic): """Return the list of hosts that have a running service for topic.""" services = db.service_get_all_by_topic(context, topic) return [service['host'] for service in services - if self.service_is_up(service)] + if utils.service_is_up(service)] def create_instance_db_entry(self, context, request_spec): """Create instance DB entry based on request_spec""" @@ -267,7 +257,7 @@ class Scheduler(object): # to the instance. if len(instance_ref['volumes']) != 0: services = db.service_get_all_by_topic(context, 'volume') - if len(services) < 1 or not self.service_is_up(services[0]): + if len(services) < 1 or not utils.service_is_up(services[0]): raise exception.VolumeServiceUnavailable() # Checking src host exists and compute node @@ -275,7 +265,7 @@ class Scheduler(object): services = db.service_get_all_compute_by_host(context, src) # Checking src host is alive. - if not self.service_is_up(services[0]): + if not utils.service_is_up(services[0]): raise exception.ComputeServiceUnavailable(host=src) def _live_migration_dest_check(self, context, instance_ref, dest, @@ -295,7 +285,7 @@ class Scheduler(object): dservice_ref = dservice_refs[0] # Checking dest host is alive. - if not self.service_is_up(dservice_ref): + if not utils.service_is_up(dservice_ref): raise exception.ComputeServiceUnavailable(host=dest) # Checking whether The host where instance is running diff --git a/nova/scheduler/filters/compute_filter.py b/nova/scheduler/filters/compute_filter.py index a31fffc6a272..3eb2f6f734bf 100644 --- a/nova/scheduler/filters/compute_filter.py +++ b/nova/scheduler/filters/compute_filter.py @@ -15,6 +15,7 @@ from nova import log as logging from nova.scheduler.filters import abstract_filter +from nova import utils LOG = logging.getLogger('nova.scheduler.filter.compute_filter') @@ -48,8 +49,11 @@ class ComputeFilter(abstract_filter.AbstractHostFilter): instance_type = filter_properties.get('instance_type') if host_state.topic != 'compute' or not instance_type: return True - capabilities = host_state.capabilities or {} + capabilities = host_state.capabilities + service = host_state.service + if not utils.service_is_up(service) or service['disabled']: + return False if not self._basic_ram_filter(host_state, instance_type): return False if not capabilities.get("enabled", True): diff --git a/nova/scheduler/filters/json_filter.py b/nova/scheduler/filters/json_filter.py index 53752df8cdf2..af17322b2e7a 100644 --- a/nova/scheduler/filters/json_filter.py +++ b/nova/scheduler/filters/json_filter.py @@ -128,14 +128,13 @@ class JsonFilter(abstract_filter.AbstractHostFilter): """Return a list of hosts that can fulfill the requirements specified in the query. """ - capabilities = host_state.capabilities or {} - if not capabilities.get("enabled", True): - return False - query = filter_properties.get('query', None) if not query: return True + # NOTE(comstud): Not checking capabilities or service for + # enabled/disabled so that a provided json filter can decide + result = self._process_filter(json.loads(query), host_state) if isinstance(result, list): # If any succeeded, include the host diff --git a/nova/scheduler/host_manager.py b/nova/scheduler/host_manager.py index 04eff39b5bfd..6996155ffd25 100644 --- a/nova/scheduler/host_manager.py +++ b/nova/scheduler/host_manager.py @@ -77,7 +77,7 @@ class HostState(object): previously used and lock down access. """ - def __init__(self, host, topic, capabilities=None): + def __init__(self, host, topic, capabilities=None, service=None): self.host = host self.topic = topic @@ -86,6 +86,9 @@ class HostState(object): if capabilities is None: capabilities = {} self.capabilities = ReadOnlyDict(capabilities.get(topic, None)) + if service is None: + service = {} + self.service = ReadOnlyDict(service) # Mutable available resources. # These will change as resources are virtually "consumed". self.free_ram_mb = 0 @@ -293,7 +296,8 @@ class HostManager(object): host = service['host'] capabilities = self.service_states.get(host, None) host_state = self.host_state_cls(host, topic, - capabilities=capabilities) + capabilities=capabilities, + service=dict(service.iteritems())) host_state.update_from_compute_node(compute) host_state_map[host] = host_state diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index fb800756dc6e..52f6be3b106f 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -26,6 +26,7 @@ from nova import flags from nova import exception from nova.scheduler import driver from nova.scheduler import chance +from nova import utils FLAGS = flags.FLAGS flags.DEFINE_integer("max_cores", 16, @@ -57,7 +58,7 @@ class SimpleScheduler(chance.ChanceScheduler): if host and context.is_admin: service = db.service_get_by_args(elevated, host, 'nova-compute') - if not self.service_is_up(service): + if not utils.service_is_up(service): raise exception.WillNotSchedule(host=host) return host @@ -79,7 +80,7 @@ class SimpleScheduler(chance.ChanceScheduler): instance_cores + instance_opts['vcpus'] > FLAGS.max_cores: msg = _("Not enough allocatable CPU cores remaining") raise exception.NoValidHost(reason=msg) - if self.service_is_up(service): + if utils.service_is_up(service) and not service['disabled']: return service['host'] msg = _("Is the appropriate service running?") raise exception.NoValidHost(reason=msg) @@ -120,7 +121,7 @@ class SimpleScheduler(chance.ChanceScheduler): zone, _x, host = availability_zone.partition(':') if host and context.is_admin: service = db.service_get_by_args(elevated, host, 'nova-volume') - if not self.service_is_up(service): + if not utils.service_is_up(service): raise exception.WillNotSchedule(host=host) driver.cast_to_volume_host(context, host, 'create_volume', volume_id=volume_id, **_kwargs) @@ -135,7 +136,7 @@ class SimpleScheduler(chance.ChanceScheduler): if volume_gigabytes + volume_ref['size'] > FLAGS.max_gigabytes: msg = _("Not enough allocatable volume gigabytes remaining") raise exception.NoValidHost(reason=msg) - if self.service_is_up(service): + if utils.service_is_up(service) and not service['disabled']: driver.cast_to_volume_host(context, service['host'], 'create_volume', volume_id=volume_id, **_kwargs) return None diff --git a/nova/scheduler/vsa.py b/nova/scheduler/vsa.py index dd17b19c6c00..7b45e93e53db 100644 --- a/nova/scheduler/vsa.py +++ b/nova/scheduler/vsa.py @@ -215,7 +215,7 @@ class VsaScheduler(simple.SimpleScheduler): zone, _x, host = availability_zone.partition(':') service = db.service_get_by_args(context.elevated(), host, 'nova-volume') - if not self.service_is_up(service): + if service['disabled'] or not utils.service_is_up(service): raise exception.WillNotSchedule(host=host) return host diff --git a/nova/tests/scheduler/fakes.py b/nova/tests/scheduler/fakes.py index 5fb60a206d82..0b8391a4d9d4 100644 --- a/nova/tests/scheduler/fakes.py +++ b/nova/tests/scheduler/fakes.py @@ -23,10 +23,14 @@ from nova.scheduler import zone_manager COMPUTE_NODES = [ - dict(id=1, local_gb=1024, memory_mb=1024, service=dict(host='host1')), - dict(id=2, local_gb=2048, memory_mb=2048, service=dict(host='host2')), - dict(id=3, local_gb=4096, memory_mb=4096, service=dict(host='host3')), - dict(id=4, local_gb=8192, memory_mb=8192, service=dict(host='host4')), + dict(id=1, local_gb=1024, memory_mb=1024, + service=dict(host='host1', disabled=False)), + dict(id=2, local_gb=2048, memory_mb=2048, + service=dict(host='host2', disabled=True)), + dict(id=3, local_gb=4096, memory_mb=4096, + service=dict(host='host3', disabled=False)), + dict(id=4, local_gb=8192, memory_mb=8192, + service=dict(host='host4', disabled=False)), # Broken entry dict(id=5, local_gb=1024, memory_mb=1024, service=None), ] diff --git a/nova/tests/scheduler/test_host_filters.py b/nova/tests/scheduler/test_host_filters.py index 40f86990226b..8462422ad199 100644 --- a/nova/tests/scheduler/test_host_filters.py +++ b/nova/tests/scheduler/test_host_filters.py @@ -1,5 +1,4 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -21,6 +20,7 @@ import json from nova.scheduler import filters from nova import test from nova.tests.scheduler import fakes +from nova import utils class HostFiltersTestCase(test.TestCase): @@ -37,64 +37,102 @@ class HostFiltersTestCase(test.TestCase): host = fakes.FakeHostState('host1', 'compute', {}) self.assertTrue(filt_cls.host_passes(host, {})) + def _stub_service_is_up(self, ret_value): + def fake_service_is_up(service): + return ret_value + self.stubs.Set(utils, 'service_is_up', fake_service_is_up) + def test_compute_filter_passes(self): + self._stub_service_is_up(True) filt_cls = filters.ComputeFilter() filter_properties = {'instance_type': {'memory_mb': 1024}} capabilities = {'enabled': True} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'compute', - {'free_ram_mb': 1024, 'capabilities': capabilities}) + {'free_ram_mb': 1024, 'capabilities': capabilities, + 'service': service}) self.assertTrue(filt_cls.host_passes(host, filter_properties)) def test_compute_filter_fails_on_memory(self): + self._stub_service_is_up(True) filt_cls = filters.ComputeFilter() filter_properties = {'instance_type': {'memory_mb': 1024}} capabilities = {'enabled': True} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'compute', - {'free_ram_mb': 1023, 'capabilities': capabilities}) + {'free_ram_mb': 1023, 'capabilities': capabilities, + 'service': service}) self.assertFalse(filt_cls.host_passes(host, filter_properties)) - def test_compute_filter_fails_on_disabled(self): + def test_compute_filter_fails_on_service_disabled(self): + self._stub_service_is_up(True) filt_cls = filters.ComputeFilter() filter_properties = {'instance_type': {'memory_mb': 1024}} - capabilities = {'enabled': False} + capabilities = {'enabled': True} + service = {'disabled': True} host = fakes.FakeHostState('host1', 'compute', - {'free_ram_mb': 1024, 'capabilities': capabilities}) + {'free_ram_mb': 1024, 'capabilities': capabilities, + 'service': service}) + self.assertFalse(filt_cls.host_passes(host, filter_properties)) + + def test_compute_filter_fails_on_service_down(self): + self._stub_service_is_up(False) + filt_cls = filters.ComputeFilter() + filter_properties = {'instance_type': {'memory_mb': 1024}} + capabilities = {'enabled': True} + service = {'disabled': False} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': 1024, 'capabilities': capabilities, + 'service': service}) self.assertFalse(filt_cls.host_passes(host, filter_properties)) def test_compute_filter_passes_on_volume(self): + self._stub_service_is_up(True) filt_cls = filters.ComputeFilter() filter_properties = {'instance_type': {'memory_mb': 1024}} capabilities = {'enabled': False} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'volume', - {'free_ram_mb': 1024, 'capabilities': capabilities}) + {'free_ram_mb': 1024, 'capabilities': capabilities, + 'service': service}) self.assertTrue(filt_cls.host_passes(host, filter_properties)) def test_compute_filter_passes_on_no_instance_type(self): + self._stub_service_is_up(True) filt_cls = filters.ComputeFilter() filter_properties = {} capabilities = {'enabled': False} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'compute', - {'free_ram_mb': 1024, 'capabilities': capabilities}) + {'free_ram_mb': 1024, 'capabilities': capabilities, + 'service': service}) self.assertTrue(filt_cls.host_passes(host, filter_properties)) def test_compute_filter_passes_extra_specs(self): + self._stub_service_is_up(True) filt_cls = filters.ComputeFilter() extra_specs = {'opt1': 1, 'opt2': 2} capabilities = {'enabled': True, 'opt1': 1, 'opt2': 2} + service = {'disabled': False} filter_properties = {'instance_type': {'memory_mb': 1024, 'extra_specs': extra_specs}} host = fakes.FakeHostState('host1', 'compute', - {'free_ram_mb': 1024, 'capabilities': capabilities}) + {'free_ram_mb': 1024, 'capabilities': capabilities, + 'service': service}) self.assertTrue(filt_cls.host_passes(host, filter_properties)) def test_compute_filter_fails_extra_specs(self): + self._stub_service_is_up(True) filt_cls = filters.ComputeFilter() extra_specs = {'opt1': 1, 'opt2': 3} capabilities = {'enabled': True, 'opt1': 1, 'opt2': 2} + service = {'disabled': False} filter_properties = {'instance_type': {'memory_mb': 1024, 'extra_specs': extra_specs}} host = fakes.FakeHostState('host1', 'compute', - {'free_ram_mb': 1024, 'capabilities': capabilities}) + {'free_ram_mb': 1024, 'capabilities': capabilities, + 'service': service}) + self.assertFalse(filt_cls.host_passes(host, filter_properties)) def test_json_filter_passes(self): @@ -156,11 +194,15 @@ class HostFiltersTestCase(test.TestCase): 'capabilities': capabilities}) self.assertFalse(filt_cls.host_passes(host, filter_properties)) - def test_json_filter_fails_on_disabled(self): + def test_json_filter_fails_on_caps_disabled(self): filt_cls = filters.JsonFilter() + json_query = json.dumps( + ['and', ['>=', '$free_ram_mb', 1024], + ['>=', '$free_disk_mb', 200 * 1024], + '$capabilities.enabled']) filter_properties = {'instance_type': {'memory_mb': 1024, 'local_gb': 200}, - 'query': self.json_query} + 'query': json_query} capabilities = {'enabled': False} host = fakes.FakeHostState('host1', 'compute', {'free_ram_mb': 1024, @@ -168,10 +210,40 @@ class HostFiltersTestCase(test.TestCase): 'capabilities': capabilities}) self.assertFalse(filt_cls.host_passes(host, filter_properties)) + def test_json_filter_fails_on_service_disabled(self): + filt_cls = filters.JsonFilter() + json_query = json.dumps( + ['and', ['>=', '$free_ram_mb', 1024], + ['>=', '$free_disk_mb', 200 * 1024], + ['not', '$service.disabled']]) + filter_properties = {'instance_type': {'memory_mb': 1024, + 'local_gb': 200}, + 'query': json_query} + capabilities = {'enabled': True} + service = {'disabled': True} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': 1024, + 'free_disk_mb': 200 * 1024, + 'capabilities': capabilities}) + self.assertFalse(filt_cls.host_passes(host, filter_properties)) + + def test_json_filter_passes(self): + filt_cls = filters.JsonFilter() + filter_properties = {'instance_type': {'memory_mb': 1024, + 'local_gb': 200}, + 'query': self.json_query} + capabilities = {'enabled': True} + host = fakes.FakeHostState('host1', 'compute', + {'free_ram_mb': 1024, + 'free_disk_mb': 200 * 1024, + 'capabilities': capabilities}) + self.assertTrue(filt_cls.host_passes(host, filter_properties)) + def test_json_filter_happy_day(self): """Test json filter more thoroughly""" filt_cls = filters.JsonFilter() raw = ['and', + '$capabilities.enabled', ['=', '$capabilities.opt1', 'match'], ['or', ['and', @@ -184,50 +256,62 @@ class HostFiltersTestCase(test.TestCase): # Passes capabilities = {'enabled': True, 'opt1': 'match'} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'compute', {'free_ram_mb': 10, 'free_disk_mb': 200, - 'capabilities': capabilities}) + 'capabilities': capabilities, + 'service': service}) self.assertTrue(filt_cls.host_passes(host, filter_properties)) # Passes capabilities = {'enabled': True, 'opt1': 'match'} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'compute', {'free_ram_mb': 40, 'free_disk_mb': 400, - 'capabilities': capabilities}) + 'capabilities': capabilities, + 'service': service}) self.assertTrue(filt_cls.host_passes(host, filter_properties)) - # Failes due to disabled + # Failes due to caps disabled capabilities = {'enabled': False, 'opt1': 'match'} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'instance_type', {'free_ram_mb': 40, 'free_disk_mb': 400, - 'capabilities': capabilities}) + 'capabilities': capabilities, + 'service': service}) self.assertFalse(filt_cls.host_passes(host, filter_properties)) # Fails due to being exact memory/disk we don't want capabilities = {'enabled': True, 'opt1': 'match'} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'compute', {'free_ram_mb': 30, 'free_disk_mb': 300, - 'capabilities': capabilities}) + 'capabilities': capabilities, + 'service': service}) self.assertFalse(filt_cls.host_passes(host, filter_properties)) # Fails due to memory lower but disk higher capabilities = {'enabled': True, 'opt1': 'match'} + service = {'disabled': False} host = fakes.FakeHostState('host1', 'compute', {'free_ram_mb': 20, 'free_disk_mb': 400, - 'capabilities': capabilities}) + 'capabilities': capabilities, + 'service': service}) self.assertFalse(filt_cls.host_passes(host, filter_properties)) # Fails due to capabilities 'opt1' not equal capabilities = {'enabled': True, 'opt1': 'no-match'} + service = {'enabled': True} host = fakes.FakeHostState('host1', 'compute', {'free_ram_mb': 20, 'free_disk_mb': 400, - 'capabilities': capabilities}) + 'capabilities': capabilities, + 'service': service}) self.assertFalse(filt_cls.host_passes(host, filter_properties)) def test_json_filter_basic_operators(self): diff --git a/nova/tests/scheduler/test_host_manager.py b/nova/tests/scheduler/test_host_manager.py index ed0fb3d63bba..5b23d09867e1 100644 --- a/nova/tests/scheduler/test_host_manager.py +++ b/nova/tests/scheduler/test_host_manager.py @@ -264,6 +264,12 @@ class HostManagerTestCase(test.TestCase): self.mox.VerifyAll() self.assertEqual(len(host_states), 4) + # Check that .service is set properly + for i in xrange(4): + compute_node = fakes.COMPUTE_NODES[i] + host = compute_node['service']['host'] + self.assertEqual(host_states[host].service, + compute_node['service']) self.assertEqual(host_states['host1'].free_ram_mb, 0) # 511GB self.assertEqual(host_states['host1'].free_disk_mb, 523264) diff --git a/nova/tests/scheduler/test_vsa_scheduler.py b/nova/tests/scheduler/test_vsa_scheduler.py index cc141897ea12..a7cd2358a347 100644 --- a/nova/tests/scheduler/test_vsa_scheduler.py +++ b/nova/tests/scheduler/test_vsa_scheduler.py @@ -197,7 +197,7 @@ class VsaSchedulerTestCase(test.TestCase): scheduled_volume = {'id': volume_id, 'host': values['host']} def _fake_service_get_by_args(self, context, host, binary): - return "service" + return {'host': 'fake_host', 'disabled': False} def _fake_service_is_up_True(self, service): return True @@ -386,7 +386,7 @@ class VsaSchedulerTestCase(test.TestCase): self.stubs.Set(nova.db, 'service_get_by_args', self._fake_service_get_by_args) - self.stubs.Set(self.sched, + self.stubs.Set(utils, 'service_is_up', self._fake_service_is_up_False) self.assertRaises(exception.WillNotSchedule, @@ -395,7 +395,7 @@ class VsaSchedulerTestCase(test.TestCase): request_spec, availability_zone="nova:host_5") - self.stubs.Set(self.sched, + self.stubs.Set(utils, 'service_is_up', self._fake_service_is_up_True) self.sched.schedule_create_volumes(self.context, @@ -462,7 +462,7 @@ class VsaSchedulerTestCase(test.TestCase): self.stubs.Set(nova.db, 'volume_get', _fake_volume_get_az) self.stubs.Set(nova.db, 'service_get_by_args', self._fake_service_get_by_args) - self.stubs.Set(self.sched, + self.stubs.Set(utils, 'service_is_up', self._fake_service_is_up_True) self.sched.schedule_create_volume(self.context, diff --git a/nova/utils.py b/nova/utils.py index 53832153edcc..b629500377e9 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -1399,3 +1399,11 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): # Install our warnings handler warnings.showwarning = _showwarning + + +def service_is_up(service): + """Check whether a service is up based on last heartbeat.""" + last_heartbeat = service['updated_at'] or service['created_at'] + # Timestamps in DB are UTC. + elapsed = total_seconds(utcnow() - last_heartbeat) + return abs(elapsed) <= FLAGS.service_down_time