From 18f3684046256529baf1d6eb9ce100c9ac3f8cac Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 7 Apr 2011 02:07:19 +0900 Subject: [PATCH 01/27] Enable RightAWS style signing on server_string without port number portion. --- nova/auth/manager.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 48684539..b51cc7f4 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -315,6 +315,15 @@ class AuthManager(object): LOG.debug('expected_signature: %s', expected_signature) LOG.debug('signature: %s', signature) if signature != expected_signature: + secondary = utils.get_secondary_server_string(server_string) + if secondary is not '': + secondary_signature = signer.Signer( + user.secret.encode()).generate(params, verb, + secondary, path) + LOG.debug('secondary_signature: %s', secondary_signature) + if signature == secondary_signature: + return (user, project) + # NOTE(itoumsn): RightAWS success case. LOG.audit(_("Invalid signature for user %s"), user.name) raise exception.NotAuthorized(_('Signature does not match')) return (user, project) From e3845e65efb06d777a89e76a4fea8a616fe908a8 Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 7 Apr 2011 11:42:02 +0900 Subject: [PATCH 02/27] pep8 cleanup. --- nova/auth/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index b51cc7f4..f1d4a1e3 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -315,7 +315,7 @@ class AuthManager(object): LOG.debug('expected_signature: %s', expected_signature) LOG.debug('signature: %s', signature) if signature != expected_signature: - secondary = utils.get_secondary_server_string(server_string) + secondary = utils.get_secondary_server_string(server_string) if secondary is not '': secondary_signature = signer.Signer( user.secret.encode()).generate(params, verb, From 1283424a7ba0ea5c2a32ecd5b13b5fe62bbf329c Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 7 Apr 2011 23:48:00 +0900 Subject: [PATCH 03/27] Blush up a bit. --- nova/auth/manager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index f1d4a1e3..c8a3a46a 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -315,15 +315,15 @@ class AuthManager(object): LOG.debug('expected_signature: %s', expected_signature) LOG.debug('signature: %s', signature) if signature != expected_signature: - secondary = utils.get_secondary_server_string(server_string) - if secondary is not '': - secondary_signature = signer.Signer( + host_only = utils.get_host_only_server_string(server_string) + # If the given server_string contains port num, try without it. + if host_only is not '': + host_only_signature = signer.Signer( user.secret.encode()).generate(params, verb, - secondary, path) - LOG.debug('secondary_signature: %s', secondary_signature) - if signature == secondary_signature: + host_only, path) + LOG.debug('host_only_signature: %s', host_only_signature) + if signature == host_only_signature: return (user, project) - # NOTE(itoumsn): RightAWS success case. LOG.audit(_("Invalid signature for user %s"), user.name) raise exception.NotAuthorized(_('Signature does not match')) return (user, project) From 8e030164de8c1582eee961af0fce1bb74d7a9b2f Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Wed, 13 Apr 2011 02:11:36 +0900 Subject: [PATCH 04/27] Blushed up a little bit. --- nova/auth/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index c8a3a46a..01aa87e3 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -317,7 +317,7 @@ class AuthManager(object): if signature != expected_signature: host_only = utils.get_host_only_server_string(server_string) # If the given server_string contains port num, try without it. - if host_only is not '': + if host_only != '': host_only_signature = signer.Signer( user.secret.encode()).generate(params, verb, host_only, path) From 3f8c4d8eef9f1efc577798ce814a22bbc762387a Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 14 Apr 2011 10:47:37 +0900 Subject: [PATCH 05/27] Updated following to RIck's comments. --- nova/auth/manager.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 01aa87e3..dc37ae06 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -300,9 +300,9 @@ class AuthManager(object): if check_type == 's3': sign = signer.Signer(user.secret.encode()) expected_signature = sign.s3_authorization(headers, verb, path) - LOG.debug('user.secret: %s', user.secret) - LOG.debug('expected_signature: %s', expected_signature) - LOG.debug('signature: %s', signature) + LOG.debug(_('user.secret: %s'), user.secret) + LOG.debug(_('expected_signature: %s'), expected_signature) + LOG.debug(_('signature: %s'), signature) if signature != expected_signature: LOG.audit(_("Invalid signature for user %s"), user.name) raise exception.NotAuthorized(_('Signature does not match')) @@ -311,9 +311,9 @@ class AuthManager(object): # secret isn't unicode expected_signature = signer.Signer(user.secret.encode()).generate( params, verb, server_string, path) - LOG.debug('user.secret: %s', user.secret) - LOG.debug('expected_signature: %s', expected_signature) - LOG.debug('signature: %s', signature) + LOG.debug(_('user.secret: %s'), user.secret) + LOG.debug(_('expected_signature: %s'), expected_signature) + LOG.debug(_('signature: %s'), signature) if signature != expected_signature: host_only = utils.get_host_only_server_string(server_string) # If the given server_string contains port num, try without it. @@ -321,7 +321,8 @@ class AuthManager(object): host_only_signature = signer.Signer( user.secret.encode()).generate(params, verb, host_only, path) - LOG.debug('host_only_signature: %s', host_only_signature) + LOG.debug(_('host_only_signature: %s'), + host_only_signature) if signature == host_only_signature: return (user, project) LOG.audit(_("Invalid signature for user %s"), user.name) From 27662ff3eddc64f69feb69c733fdb2e6d081864b Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Fri, 22 Apr 2011 01:26:59 +0900 Subject: [PATCH 06/27] Utility method reworked, etc. --- nova/auth/authutils.py | 48 +++++++++++++++++++++++++++++++++++++++++ nova/auth/manager.py | 4 +++- nova/tests/test_auth.py | 24 +++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 nova/auth/authutils.py diff --git a/nova/auth/authutils.py b/nova/auth/authutils.py new file mode 100644 index 00000000..429e86ef --- /dev/null +++ b/nova/auth/authutils.py @@ -0,0 +1,48 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 NTT DATA CORPORATION. +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Auth module specific utilities and helper functions. +""" + +import netaddr +import string + + +def get_host_only_server_string(server_str): + """ + Returns host part only of the given server_string if it's a combination + of host part and port. Otherwise, return null string. + """ + + # First of all, exclude pure IPv6 address (w/o port). + if netaddr.valid_ipv6(server_str): + return '' + + # Next, check if this is IPv6 address with port number combination. + if server_str.find("]:") != -1: + [address, sep, port] = server_str.replace('[', '', 1).partition(']:') + return address + + # Third, check if this is a combination of general address and port + if server_str.find(':') == -1: + return '' + + # This must be a combination of host part and port + (address, port) = server_str.split(':') + return address diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 06def220..775b38af 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -35,6 +35,7 @@ from nova import flags from nova import log as logging from nova import utils from nova.auth import signer +from nova.auth import authutils FLAGS = flags.FLAGS @@ -315,7 +316,8 @@ class AuthManager(object): LOG.debug(_('expected_signature: %s'), expected_signature) LOG.debug(_('signature: %s'), signature) if signature != expected_signature: - host_only = utils.get_host_only_server_string(server_string) + host_only = authutils.get_host_only_server_string( + server_string) # If the given server_string contains port num, try without it. if host_only != '': host_only_signature = signer.Signer( diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index f8a1b156..3886e9e6 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -25,6 +25,7 @@ from nova import log as logging from nova import test from nova.auth import manager from nova.api.ec2 import cloud +from nova.auth import authutils FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.auth_unittest') @@ -339,6 +340,29 @@ class AuthManagerDbTestCase(_AuthManagerBaseTestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' +class AuthManagerUtilTestCase(test.TestCase): + def test_get_host_only_server_string(self): + result = authutils.get_host_only_server_string('::1') + self.assertEqual('', result) + result = authutils.get_host_only_server_string('[::1]:8773') + self.assertEqual('::1', result) + result = authutils.get_host_only_server_string('2001:db8::192.168.1.1') + self.assertEqual('', result) + result = authutils.get_host_only_server_string( + '[2001:db8::192.168.1.1]:8773') + self.assertEqual('2001:db8::192.168.1.1', result) + result = authutils.get_host_only_server_string('192.168.1.1') + self.assertEqual('', result) + result = authutils.get_host_only_server_string('192.168.1.2:8773') + self.assertEqual('192.168.1.2', result) + result = authutils.get_host_only_server_string('192.168.1.3') + self.assertEqual('', result) + result = authutils.get_host_only_server_string('www.example.com:8443') + self.assertEqual('www.example.com', result) + result = authutils.get_host_only_server_string('www.example.com') + self.assertEqual('', result) + + if __name__ == "__main__": # TODO: Implement use_fake as an option unittest.main() From 4683480df0a22e5ddb351dfd6a51df0e93395051 Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Fri, 22 Apr 2011 21:35:54 +0900 Subject: [PATCH 07/27] Rework completed. Added test cases, changed helper method name, etc. --- nova/auth/authutils.py | 48 ------------------------------- nova/auth/manager.py | 8 ++---- nova/tests/test_auth.py | 64 ++++++++++++++++++++++++----------------- 3 files changed, 40 insertions(+), 80 deletions(-) delete mode 100644 nova/auth/authutils.py diff --git a/nova/auth/authutils.py b/nova/auth/authutils.py deleted file mode 100644 index 429e86ef..00000000 --- a/nova/auth/authutils.py +++ /dev/null @@ -1,48 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 NTT DATA CORPORATION. -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Auth module specific utilities and helper functions. -""" - -import netaddr -import string - - -def get_host_only_server_string(server_str): - """ - Returns host part only of the given server_string if it's a combination - of host part and port. Otherwise, return null string. - """ - - # First of all, exclude pure IPv6 address (w/o port). - if netaddr.valid_ipv6(server_str): - return '' - - # Next, check if this is IPv6 address with port number combination. - if server_str.find("]:") != -1: - [address, sep, port] = server_str.replace('[', '', 1).partition(']:') - return address - - # Third, check if this is a combination of general address and port - if server_str.find(':') == -1: - return '' - - # This must be a combination of host part and port - (address, port) = server_str.split(':') - return address diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 775b38af..d42594c8 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -35,7 +35,6 @@ from nova import flags from nova import log as logging from nova import utils from nova.auth import signer -from nova.auth import authutils FLAGS = flags.FLAGS @@ -316,13 +315,12 @@ class AuthManager(object): LOG.debug(_('expected_signature: %s'), expected_signature) LOG.debug(_('signature: %s'), signature) if signature != expected_signature: - host_only = authutils.get_host_only_server_string( - server_string) + (addr_str, port_str) = utils.parse_server_string(server_string) # If the given server_string contains port num, try without it. - if host_only != '': + if port_str != '': host_only_signature = signer.Signer( user.secret.encode()).generate(params, verb, - host_only, path) + addr_str, path) LOG.debug(_('host_only_signature: %s'), host_only_signature) if signature == host_only_signature: diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 3886e9e6..f02dd94b 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -25,7 +25,6 @@ from nova import log as logging from nova import test from nova.auth import manager from nova.api.ec2 import cloud -from nova.auth import authutils FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.auth_unittest') @@ -102,9 +101,43 @@ class _AuthManagerBaseTestCase(test.TestCase): self.assertEqual('private-party', u.access) def test_004_signature_is_valid(self): - #self.assertTrue(self.manager.authenticate(**boto.generate_url ...? )) - pass - #raise NotImplementedError + with user_generator(self.manager, name='admin', secret='admin', + access='admin'): + with project_generator(self.manager, name="admin", + manager_user='admin'): + accesskey = 'admin:admin' + expected_result = (self.manager.get_user('admin'), + self.manager.get_project('admin')) + # captured sig and query string using boto 1.9b/euca2ools 1.2 + sig = 'd67Wzd9Bwz8xid9QU+lzWXcF2Y3tRicYABPJgrqfrwM=' + auth_params = {'AWSAccessKeyId': 'admin:admin', + 'Action': 'DescribeAvailabilityZones', + 'SignatureMethod': 'HmacSHA256', + 'SignatureVersion': '2', + 'Timestamp': '2011-04-22T11:29:29', + 'Version': '2009-11-30'} + self.assertTrue(expected_result, self.manager.authenticate( + accesskey, + sig, + auth_params, + 'GET', + '127.0.0.1:8773', + '/services/Cloud/')) + # captured sig and query string using RightAWS 1.10.0 + sig = 'ECYLU6xdFG0ZqRVhQybPJQNJ5W4B9n8fGs6+/fuGD2c=' + auth_params = {'AWSAccessKeyId': 'admin:admin', + 'Action': 'DescribeAvailabilityZones', + 'SignatureMethod': 'HmacSHA256', + 'SignatureVersion': '2', + 'Timestamp': '2011-04-22T11:29:49.000Z', + 'Version': '2008-12-01'} + self.assertTrue(expected_result, self.manager.authenticate( + accesskey, + sig, + auth_params, + 'GET', + '127.0.0.1', + '/services/Cloud')) def test_005_can_get_credentials(self): return @@ -340,29 +373,6 @@ class AuthManagerDbTestCase(_AuthManagerBaseTestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' -class AuthManagerUtilTestCase(test.TestCase): - def test_get_host_only_server_string(self): - result = authutils.get_host_only_server_string('::1') - self.assertEqual('', result) - result = authutils.get_host_only_server_string('[::1]:8773') - self.assertEqual('::1', result) - result = authutils.get_host_only_server_string('2001:db8::192.168.1.1') - self.assertEqual('', result) - result = authutils.get_host_only_server_string( - '[2001:db8::192.168.1.1]:8773') - self.assertEqual('2001:db8::192.168.1.1', result) - result = authutils.get_host_only_server_string('192.168.1.1') - self.assertEqual('', result) - result = authutils.get_host_only_server_string('192.168.1.2:8773') - self.assertEqual('192.168.1.2', result) - result = authutils.get_host_only_server_string('192.168.1.3') - self.assertEqual('', result) - result = authutils.get_host_only_server_string('www.example.com:8443') - self.assertEqual('www.example.com', result) - result = authutils.get_host_only_server_string('www.example.com') - self.assertEqual('', result) - - if __name__ == "__main__": # TODO: Implement use_fake as an option unittest.main() From 19a4d66df07329324b459a25928b3cacabffbffe Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 2 May 2011 12:49:10 -0700 Subject: [PATCH 08/27] initial pass --- nova/scheduler/query.py | 164 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 nova/scheduler/query.py diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py new file mode 100644 index 00000000..5379b3d4 --- /dev/null +++ b/nova/scheduler/query.py @@ -0,0 +1,164 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Query is a plug-in mechanism for requesting instance resources. +Three plug-ins are included: PassThru, Flavor & JSON. PassThru just +returns the full, unfiltered list of hosts. Flavor is a hard coded +matching mechanism based on flavor criteria and JSON is an ad-hoc +query grammar. +""" + +from nova import exception +from nova import utils + +class Query: + """Base class for query plug-ins.""" + + def instance_type_to_query(self, instance_type): + """Convert instance_type into a query for most common use-case.""" + raise exception.Error(_("Query driver not specified.")) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that fulfill the query.""" + raise exception.Error(_("Query driver not specified.")) + + +def load_driver(driver_name): + resource = utils.import_class(driver_name) + if type(resource) != types.ClassType: + continue + cls = resource + if not issubclass(cls, Query): + raise exception.Error(_("Query driver does not derive " + "from nova.scheduler.query.Query.")) + return cls + + +class PassThruQuery: + """NOP query plug-in. Returns all hosts in ZoneManager. + This essentially does what the old Scheduler+Chance used + to give us.""" + + def instance_type_to_query(self, instance_type): + """Return anything to prevent base-class from raising + exception.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts from ZoneManager list.""" + hosts = zone_manager.service_state.get('compute', {}) + return [(host, capabilities) + for host, capabilities in hosts.iteritems()] + + +class FlavorQuery: + """Query plug-in hard-coded to work with flavors.""" + + def instance_type_to_query(self, instance_type): + """Use instance_type to filter hosts.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can create instance_type.""" + hosts = zone_manager.service_state.get('compute', {}) + selected_hosts = [] + instance_type = query + for host, capabilities in hosts.iteritems(): + host_ram_mb = capabilities.get['host_memory']['free'] + disk_bytes = capabilities.get['disk']['available'] + if host_ram_mb >= instance_type['memory_mb'] and \ + disk_bytes >= instance_type['local_gb']: + selected_hosts.append((host, capabilities)) + return selected_hosts + +#host entries (currently) are like: +# {'host_name-description': 'Default install of XenServer', +# 'host_hostname': 'xs-mini', +# 'host_memory': {'total': 8244539392, +# 'overhead': 184225792, +# 'free': 3868327936, +# 'free-computed': 3840843776}, +# 'host_other-config': {}, +# 'host_ip_address': '192.168.1.109', +# 'host_cpu_info': {}, +# 'disk': {'available': 32954957824, +# 'total': 50394562560, +# 'used': 17439604736}, +# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', +# 'host_name-label': 'xs-mini'} + +# instance_type table has: +#name = Column(String(255), unique=True) +#memory_mb = Column(Integer) +#vcpus = Column(Integer) +#local_gb = Column(Integer) +#flavorid = Column(Integer, unique=True) +#swap = Column(Integer, nullable=False, default=0) +#rxtx_quota = Column(Integer, nullable=False, default=0) +#rxtx_cap = Column(Integer, nullable=False, default=0) + +class JsonQuery: + """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" + + def _equals(self, args): + pass + + def _less_than(self, args): + pass + + def _greater_than(self, args): + pass + + def _in(self, args): + pass + + def _less_than_equal(self, args): + pass + + def _greater_than_equal(self, args): + pass + + def _not(self, args): + pass + + def _must(self, args): + pass + + def _or(self, args): + pass + + commands = { + '=': _equals, + '<': _less_than, + '>': _greater_than, + 'in': _in, + '<=': _less_than_equal, + '>=': _greater_than_equal, + 'not': _not, + 'must', _must, + 'or', _or, + } + + def instance_type_to_query(self, instance_type): + """Convert instance_type into JSON query object.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can fulfill query.""" + return [] + + + From 68977a27abc6d881bffefcf7077e41365dc32041 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 3 May 2011 15:02:55 -0700 Subject: [PATCH 09/27] tests and better driver loading --- nova/scheduler/query.py | 50 +++++++++++++--------- nova/tests/test_query.py | 89 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 nova/tests/test_query.py diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 5379b3d4..4971bc7e 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -15,39 +15,38 @@ """ Query is a plug-in mechanism for requesting instance resources. -Three plug-ins are included: PassThru, Flavor & JSON. PassThru just +Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just returns the full, unfiltered list of hosts. Flavor is a hard coded matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. """ from nova import exception +from nova import flags +from nova import log as logging from nova import utils +LOG = logging.getLogger('nova.scheduler.query') + +FLAGS = flags.FLAGS +flags.DEFINE_string('default_query_engine', + 'nova.scheduler.query.AllHostsQuery', + 'Which query engine to use for filtering hosts.') + + class Query: """Base class for query plug-ins.""" def instance_type_to_query(self, instance_type): """Convert instance_type into a query for most common use-case.""" - raise exception.Error(_("Query driver not specified.")) + raise exception.BadSchedulerQueryDriver() def filter_hosts(self, zone_manager, query): """Return a list of hosts that fulfill the query.""" - raise exception.Error(_("Query driver not specified.")) + raise exception.BadSchedulerQueryDriver() -def load_driver(driver_name): - resource = utils.import_class(driver_name) - if type(resource) != types.ClassType: - continue - cls = resource - if not issubclass(cls, Query): - raise exception.Error(_("Query driver does not derive " - "from nova.scheduler.query.Query.")) - return cls - - -class PassThruQuery: +class AllHostsQuery: """NOP query plug-in. Returns all hosts in ZoneManager. This essentially does what the old Scheduler+Chance used to give us.""" @@ -59,7 +58,7 @@ class PassThruQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" - hosts = zone_manager.service_state.get('compute', {}) + hosts = zone_manager.service_states.get('compute', {}) return [(host, capabilities) for host, capabilities in hosts.iteritems()] @@ -73,7 +72,7 @@ class FlavorQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" - hosts = zone_manager.service_state.get('compute', {}) + hosts = zone_manager.service_states.get('compute', {}) selected_hosts = [] instance_type = query for host, capabilities in hosts.iteritems(): @@ -148,8 +147,8 @@ class JsonQuery: '<=': _less_than_equal, '>=': _greater_than_equal, 'not': _not, - 'must', _must, - 'or', _or, + 'must': _must, + 'or': _or, } def instance_type_to_query(self, instance_type): @@ -161,4 +160,15 @@ class JsonQuery: return [] - +# Since the caller may specify which driver to use we need +# to have an authoritative list of what is permissible. +DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] + + +def choose_driver(driver_name=None): + if not driver_name: + driver_name = FLAGS.default_query_engine + for driver in DRIVERS: + if str(driver) == driver_name: + return driver + raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py new file mode 100644 index 00000000..0fab9193 --- /dev/null +++ b/nova/tests/test_query.py @@ -0,0 +1,89 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Scheduler Query Drivers +""" + +from nova import exception +from nova import flags +from nova import test +from nova.scheduler import query + +FLAGS = flags.FLAGS + +class FakeZoneManager: + pass + +class QueryTestCase(test.TestCase): + """Test case for query drivers.""" + + def _host_caps(self, multiplier): + return {'host_name-description':'XenServer %s' % multiplier, + 'host_hostname':'xs-%s' % multiplier, + 'host_memory':{'total': 100, + 'overhead': 5, + 'free': 5 + multiplier * 5, + 'free-computed': 5 + multiplier * 5}, + 'host_other-config':{}, + 'host_ip_address':'192.168.1.%d' % (100 + multiplier), + 'host_cpu_info':{}, + 'disk':{'available': 100 + multiplier * 100, + 'total': 1000, + 'used': 50 + multiplier * 50}, + 'host_uuid':'xxx-%d' % multiplier, + 'host_name-label':'xs-%s' % multiplier} + + def setUp(self): + self.old_flag = FLAGS.default_query_engine + FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' + self.instance_type = dict(name='tiny', + memory_mb=500, + vcpus=10, + local_gb=50, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) + + hosts = {} + for x in xrange(10): + hosts['host%s' % x] = self._host_caps(x) + + self.zone_manager = FakeZoneManager() + self.zone_manager.service_states = {} + self.zone_manager.service_states['compute'] = hosts + + def tearDown(self): + FLAGS.default_query_engine = self.old_flag + + def test_choose_driver(self): + driver = query.choose_driver() + self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + driver = query.choose_driver('nova.scheduler.query.FlavorQuery') + self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + try: + query.choose_driver('does not exist') + self.fail("Should not find driver") + except exception.SchedulerQueryDriverNotFound: + pass + + def test_all_host_driver(self): + driver = query.AllHostsQuery() + cooked = driver.instance_type_to_query(self.instance_type) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(10, len(hosts)) + for host, capabilities in hosts: + self.assertTrue(host.startswith('host')) + From c384e7f4f696dd2050442ae23c307dd9ea7b04dc Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 4 May 2011 06:16:33 -0700 Subject: [PATCH 10/27] flavor test --- nova/scheduler/query.py | 6 +++--- nova/tests/test_query.py | 44 +++++++++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 4971bc7e..1c593f1b 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -74,10 +74,10 @@ class FlavorQuery: """Return a list of hosts that can create instance_type.""" hosts = zone_manager.service_states.get('compute', {}) selected_hosts = [] - instance_type = query + query_type, instance_type = query for host, capabilities in hosts.iteritems(): - host_ram_mb = capabilities.get['host_memory']['free'] - disk_bytes = capabilities.get['disk']['available'] + host_ram_mb = capabilities['host_memory']['free'] + disk_bytes = capabilities['disk']['available'] if host_ram_mb >= instance_type['memory_mb'] and \ disk_bytes >= instance_type['local_gb']: selected_hosts.append((host, capabilities)) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 0fab9193..d89183e9 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -30,32 +30,39 @@ class QueryTestCase(test.TestCase): """Test case for query drivers.""" def _host_caps(self, multiplier): + # Returns host capabilities in the following way: + # host0 = memory:free 10 (100max) + # disk:available 100 (1000max) + # hostN = memory:free 10 + 10N + # disk:available 100 + 100N + # in other words: hostN has more resources than host0 + # which means ... don't go above 10 hosts. return {'host_name-description':'XenServer %s' % multiplier, 'host_hostname':'xs-%s' % multiplier, 'host_memory':{'total': 100, - 'overhead': 5, - 'free': 5 + multiplier * 5, - 'free-computed': 5 + multiplier * 5}, + 'overhead': 10, + 'free': 10 + multiplier * 10, + 'free-computed': 10 + multiplier * 10}, 'host_other-config':{}, 'host_ip_address':'192.168.1.%d' % (100 + multiplier), 'host_cpu_info':{}, 'disk':{'available': 100 + multiplier * 100, 'total': 1000, - 'used': 50 + multiplier * 50}, + 'used': 0}, 'host_uuid':'xxx-%d' % multiplier, 'host_name-label':'xs-%s' % multiplier} def setUp(self): self.old_flag = FLAGS.default_query_engine FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' - self.instance_type = dict(name='tiny', - memory_mb=500, - vcpus=10, - local_gb=50, - flavorid=1, - swap=500, - rxtx_quota=30000, - rxtx_cap=200) + self.instance_type = dict(name= 'tiny', + memory_mb= 50, + vcpus= 10, + local_gb= 500, + flavorid= 1, + swap= 500, + rxtx_quota= 30000, + rxtx_cap= 200) hosts = {} for x in xrange(10): @@ -69,10 +76,13 @@ class QueryTestCase(test.TestCase): FLAGS.default_query_engine = self.old_flag def test_choose_driver(self): + # Test default driver ... driver = query.choose_driver() self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + # Test valid driver ... driver = query.choose_driver('nova.scheduler.query.FlavorQuery') self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + # Test invalid driver ... try: query.choose_driver('does not exist') self.fail("Should not find driver") @@ -87,3 +97,13 @@ class QueryTestCase(test.TestCase): for host, capabilities in hosts: self.assertTrue(host.startswith('host')) + def test_flavor_driver(self): + driver = query.FlavorQuery() + # filter all hosts that can support 50 ram and 500 disk + cooked = driver.instance_type_to_query(self.instance_type) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host4', just_hosts[0]) + self.assertEquals('host9', just_hosts[5]) From c47c40b5bc4111b4fa80583b926c7888fef49762 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 4 May 2011 14:48:23 -0700 Subject: [PATCH 11/27] json parser --- nova/scheduler/query.py | 123 +++++++++++++++++++++++++++++++++------ nova/tests/test_query.py | 25 +++++--- 2 files changed, 124 insertions(+), 24 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1c593f1b..1b84e6b1 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -21,6 +21,8 @@ matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. """ +import json + from nova import exception from nova import flags from nova import log as logging @@ -58,9 +60,8 @@ class AllHostsQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" - hosts = zone_manager.service_states.get('compute', {}) - return [(host, capabilities) - for host, capabilities in hosts.iteritems()] + return [(host, services) + for host, services in zone_manager.service_state.iteritems()] class FlavorQuery: @@ -72,10 +73,10 @@ class FlavorQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" - hosts = zone_manager.service_states.get('compute', {}) + instance_type = query selected_hosts = [] - query_type, instance_type = query - for host, capabilities in hosts.iteritems(): + for host, services in zone_manager.service_states.iteritems(): + capabilities = services.get('compute', {}) host_ram_mb = capabilities['host_memory']['free'] disk_bytes = capabilities['disk']['available'] if host_ram_mb >= instance_type['memory_mb'] and \ @@ -113,31 +114,74 @@ class JsonQuery: """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" def _equals(self, args): - pass + """First term is == all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs != rhs: + return False + return True def _less_than(self, args): - pass + """First term is < all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs >= rhs: + return False + return True def _greater_than(self, args): - pass + """First term is > all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs <= rhs: + return False + return True def _in(self, args): - pass + """First term is in set of remaining terms""" + if len(args) < 2: + return False + return args[0] in args[1:] def _less_than_equal(self, args): - pass + """First term is <= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs > rhs: + return False + return True def _greater_than_equal(self, args): - pass + """First term is >= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs < rhs: + return False + return True def _not(self, args): - pass + if len(args) == 0: + return False + return not args[0] def _must(self, args): - pass + return True def _or(self, args): - pass + return True in args + + def _and(self, args): + return False not in args commands = { '=': _equals, @@ -149,15 +193,60 @@ class JsonQuery: 'not': _not, 'must': _must, 'or': _or, + 'and': _and, } def instance_type_to_query(self, instance_type): """Convert instance_type into JSON query object.""" - return (str(self.__class__), instance_type) + required_ram = instance_type['memory_mb'] + required_disk = instance_type['local_gb'] + query = ['and', + ['>=', '$compute.host_memory.free', required_ram], + ['>=', '$compute.disk.available', required_disk] + ] + return (str(self.__class__), json.dumps(query)) + + def _parse_string(self, string, host, services): + """Strings prefixed with $ are capability lookups in the + form '$service.capability[.subcap*]'""" + if not string: + return None + if string[0] != '$': + return string + + path = string[1:].split('.') + for item in path: + services = services.get(item, None) + if not services: + return None + return services + + def _process_query(self, zone_manager, query, host, services): + if len(query) == 0: + return True + cmd = query[0] + method = self.commands[cmd] # Let exception fly. + cooked_args = [] + for arg in query[1:]: + if isinstance(arg, list): + arg = self._process_query(zone_manager, arg, host, services) + elif isinstance(arg, basestring): + arg = self._parse_string(arg, host, services) + if arg != None: + cooked_args.append(arg) + result = method(self, cooked_args) + print "*** %s %s = %s" % (cmd, cooked_args, result) + return result def filter_hosts(self, zone_manager, query): """Return a list of hosts that can fulfill query.""" - return [] + expanded = json.loads(query) + hosts = [] + for host, services in zone_manager.service_states.iteritems(): + print "-----" + if self._process_query(zone_manager, expanded, host, services): + hosts.append((host, services)) + return hosts # Since the caller may specify which driver to use we need diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index d89183e9..7a036a4d 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -64,13 +64,11 @@ class QueryTestCase(test.TestCase): rxtx_quota= 30000, rxtx_cap= 200) - hosts = {} - for x in xrange(10): - hosts['host%s' % x] = self._host_caps(x) - self.zone_manager = FakeZoneManager() - self.zone_manager.service_states = {} - self.zone_manager.service_states['compute'] = hosts + states = {} + for x in xrange(10): + states['host%s' % x] = {'compute': self._host_caps(x)} + self.zone_manager.service_states = states def tearDown(self): FLAGS.default_query_engine = self.old_flag @@ -100,7 +98,20 @@ class QueryTestCase(test.TestCase): def test_flavor_driver(self): driver = query.FlavorQuery() # filter all hosts that can support 50 ram and 500 disk - cooked = driver.instance_type_to_query(self.instance_type) + name, cooked = driver.instance_type_to_query(self.instance_type) + self.assertEquals('nova.scheduler.query.FlavorQuery', name) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host4', just_hosts[0]) + self.assertEquals('host9', just_hosts[5]) + + def test_json_driver(self): + driver = query.JsonQuery() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = driver.instance_type_to_query(self.instance_type) + self.assertEquals('nova.scheduler.query.JsonQuery', name) hosts = driver.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] From 083b65bb2ecc1fc2e2021d76e04ea6cd860763cf Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 04:57:25 -0700 Subject: [PATCH 12/27] and or test --- nova/scheduler/query.py | 10 +++++----- nova/tests/test_query.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1b84e6b1..3233cc0d 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -19,6 +19,10 @@ Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just returns the full, unfiltered list of hosts. Flavor is a hard coded matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. + +Note: These are hard filters. All capabilities used must be present +or the host will excluded. If you want soft filters use the weighting +mechanism which is intended for the more touchy-feely capabilities. """ import json @@ -61,7 +65,7 @@ class AllHostsQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" return [(host, services) - for host, services in zone_manager.service_state.iteritems()] + for host, services in zone_manager.service_states.iteritems()] class FlavorQuery: @@ -174,9 +178,6 @@ class JsonQuery: return False return not args[0] - def _must(self, args): - return True - def _or(self, args): return True in args @@ -191,7 +192,6 @@ class JsonQuery: '<=': _less_than_equal, '>=': _greater_than_equal, 'not': _not, - 'must': _must, 'or': _or, 'and': _and, } diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 7a036a4d..896b2364 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -16,6 +16,8 @@ Tests For Scheduler Query Drivers """ +import json + from nova import exception from nova import flags from nova import test @@ -118,3 +120,27 @@ class QueryTestCase(test.TestCase): just_hosts.sort() self.assertEquals('host4', just_hosts[0]) self.assertEquals('host9', just_hosts[5]) + + # Try some custom queries + + raw = ['or', + ['and', + ['<', '$compute.host_memory.free', 30], + ['<', '$compute.disk.available', 300] + ], + ['and', + ['>', '$compute.host_memory.free', 70], + ['>', '$compute.disk.available', 700] + ] + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host0', just_hosts[0]) + self.assertEquals('host1', just_hosts[1]) + self.assertEquals('host7', just_hosts[2]) + self.assertEquals('host8', just_hosts[3]) + self.assertEquals('host9', just_hosts[4]) From 66c26e2849ec672061841241c271d61e11e4546f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 04:59:26 -0700 Subject: [PATCH 13/27] and or test --- nova/tests/test_query.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 896b2364..90ae80dc 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -139,8 +139,5 @@ class QueryTestCase(test.TestCase): self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host0', just_hosts[0]) - self.assertEquals('host1', just_hosts[1]) - self.assertEquals('host7', just_hosts[2]) - self.assertEquals('host8', just_hosts[3]) - self.assertEquals('host9', just_hosts[4]) + for index, host in zip([0, 1, 7, 8, 9], just_hosts): + self.assertEquals('host%d' % index, host) From a7e02d013b4843085d7c6ea45f1e97797fd563f1 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 05:29:31 -0700 Subject: [PATCH 14/27] not = --- nova/scheduler/query.py | 7 +++++-- nova/tests/test_query.py | 29 +++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 3233cc0d..0279efc9 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -176,7 +176,7 @@ class JsonQuery: def _not(self, args): if len(args) == 0: return False - return not args[0] + return [not arg for arg in args] def _or(self, args): return True in args @@ -244,7 +244,10 @@ class JsonQuery: hosts = [] for host, services in zone_manager.service_states.iteritems(): print "-----" - if self._process_query(zone_manager, expanded, host, services): + r = self._process_query(zone_manager, expanded, host, services) + if isinstance(r, list): + r = True in r + if r: hosts.append((host, services)) return hosts diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 90ae80dc..9bfc83b7 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -33,7 +33,7 @@ class QueryTestCase(test.TestCase): def _host_caps(self, multiplier): # Returns host capabilities in the following way: - # host0 = memory:free 10 (100max) + # host1 = memory:free 10 (100max) # disk:available 100 (1000max) # hostN = memory:free 10 + 10N # disk:available 100 + 100N @@ -69,7 +69,7 @@ class QueryTestCase(test.TestCase): self.zone_manager = FakeZoneManager() states = {} for x in xrange(10): - states['host%s' % x] = {'compute': self._host_caps(x)} + states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} self.zone_manager.service_states = states def tearDown(self): @@ -106,8 +106,8 @@ class QueryTestCase(test.TestCase): self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host4', just_hosts[0]) - self.assertEquals('host9', just_hosts[5]) + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) def test_json_driver(self): driver = query.JsonQuery() @@ -118,8 +118,8 @@ class QueryTestCase(test.TestCase): self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host4', just_hosts[0]) - self.assertEquals('host9', just_hosts[5]) + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) # Try some custom queries @@ -139,5 +139,18 @@ class QueryTestCase(test.TestCase): self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - for index, host in zip([0, 1, 7, 8, 9], just_hosts): - self.assertEquals('host%d' % index, host) + for index, host in zip([1, 2, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['not', + ['=', '$compute.host_memory.free', 30], + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(9, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + From 9e7ca7a1164f1b1984368d86e5eda9e616f9d037 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 05:30:58 -0700 Subject: [PATCH 15/27] not = --- nova/tests/test_query.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 9bfc83b7..78e8ee9d 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -153,4 +153,14 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) + + raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100]] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([2, 4, 6, 8, 10], just_hosts): + self.assertEquals('host%02d' % index, host) From cef477c10abc381e1b2b4054d89d099a102dbcff Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 06:01:56 -0700 Subject: [PATCH 16/27] pep8 --- nova/scheduler/query.py | 43 ++++++++++++------------ nova/tests/test_query.py | 71 ++++++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 56 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 0279efc9..4332f43b 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -114,6 +114,7 @@ class FlavorQuery: #rxtx_quota = Column(Integer, nullable=False, default=0) #rxtx_cap = Column(Integer, nullable=False, default=0) + class JsonQuery: """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" @@ -162,7 +163,7 @@ class JsonQuery: if lhs > rhs: return False return True - + def _greater_than_equal(self, args): """First term is >= all the other terms.""" if len(args) < 2: @@ -180,10 +181,10 @@ class JsonQuery: def _or(self, args): return True in args - + def _and(self, args): return False not in args - + commands = { '=': _equals, '<': _less_than, @@ -200,7 +201,7 @@ class JsonQuery: """Convert instance_type into JSON query object.""" required_ram = instance_type['memory_mb'] required_disk = instance_type['local_gb'] - query = ['and', + query = ['and', ['>=', '$compute.host_memory.free', required_ram], ['>=', '$compute.disk.available', required_disk] ] @@ -219,24 +220,24 @@ class JsonQuery: services = services.get(item, None) if not services: return None - return services + return services def _process_query(self, zone_manager, query, host, services): - if len(query) == 0: - return True - cmd = query[0] - method = self.commands[cmd] # Let exception fly. - cooked_args = [] - for arg in query[1:]: - if isinstance(arg, list): - arg = self._process_query(zone_manager, arg, host, services) - elif isinstance(arg, basestring): - arg = self._parse_string(arg, host, services) - if arg != None: - cooked_args.append(arg) - result = method(self, cooked_args) - print "*** %s %s = %s" % (cmd, cooked_args, result) - return result + if len(query) == 0: + return True + cmd = query[0] + method = self.commands[cmd] # Let exception fly. + cooked_args = [] + for arg in query[1:]: + if isinstance(arg, list): + arg = self._process_query(zone_manager, arg, host, services) + elif isinstance(arg, basestring): + arg = self._parse_string(arg, host, services) + if arg != None: + cooked_args.append(arg) + result = method(self, cooked_args) + print "*** %s %s = %s" % (cmd, cooked_args, result) + return result def filter_hosts(self, zone_manager, query): """Return a list of hosts that can fulfill query.""" @@ -253,7 +254,7 @@ class JsonQuery: # Since the caller may specify which driver to use we need -# to have an authoritative list of what is permissible. +# to have an authoritative list of what is permissible. DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 78e8ee9d..8b894a4e 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -25,9 +25,11 @@ from nova.scheduler import query FLAGS = flags.FLAGS + class FakeZoneManager: pass + class QueryTestCase(test.TestCase): """Test case for query drivers.""" @@ -39,32 +41,32 @@ class QueryTestCase(test.TestCase): # disk:available 100 + 100N # in other words: hostN has more resources than host0 # which means ... don't go above 10 hosts. - return {'host_name-description':'XenServer %s' % multiplier, - 'host_hostname':'xs-%s' % multiplier, - 'host_memory':{'total': 100, + return {'host_name-description': 'XenServer %s' % multiplier, + 'host_hostname': 'xs-%s' % multiplier, + 'host_memory': {'total': 100, 'overhead': 10, 'free': 10 + multiplier * 10, 'free-computed': 10 + multiplier * 10}, - 'host_other-config':{}, - 'host_ip_address':'192.168.1.%d' % (100 + multiplier), - 'host_cpu_info':{}, - 'disk':{'available': 100 + multiplier * 100, + 'host_other-config': {}, + 'host_ip_address': '192.168.1.%d' % (100 + multiplier), + 'host_cpu_info': {}, + 'disk': {'available': 100 + multiplier * 100, 'total': 1000, 'used': 0}, - 'host_uuid':'xxx-%d' % multiplier, - 'host_name-label':'xs-%s' % multiplier} + 'host_uuid': 'xxx-%d' % multiplier, + 'host_name-label': 'xs-%s' % multiplier} def setUp(self): self.old_flag = FLAGS.default_query_engine FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' - self.instance_type = dict(name= 'tiny', - memory_mb= 50, - vcpus= 10, - local_gb= 500, - flavorid= 1, - swap= 500, - rxtx_quota= 30000, - rxtx_cap= 200) + self.instance_type = dict(name='tiny', + memory_mb=50, + vcpus=10, + local_gb=500, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) self.zone_manager = FakeZoneManager() states = {} @@ -123,16 +125,16 @@ class QueryTestCase(test.TestCase): # Try some custom queries - raw = ['or', - ['and', - ['<', '$compute.host_memory.free', 30], - ['<', '$compute.disk.available', 300] - ], - ['and', - ['>', '$compute.host_memory.free', 70], - ['>', '$compute.disk.available', 700] - ] - ] + raw = ['or', + ['and', + ['<', '$compute.host_memory.free', 30], + ['<', '$compute.disk.available', 300] + ], + ['and', + ['>', '$compute.host_memory.free', 70], + ['>', '$compute.disk.available', 700] + ] + ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -141,10 +143,10 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - - raw = ['not', - ['=', '$compute.host_memory.free', 30], - ] + + raw = ['not', + ['=', '$compute.host_memory.free', 30], + ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -153,8 +155,8 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - - raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100]] + + raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -163,4 +165,3 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([2, 4, 6, 8, 10], just_hosts): self.assertEquals('host%02d' % index, host) - From 9d91d098fc80e445470c57bb6e603489b1ca0eb3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 07:35:44 -0700 Subject: [PATCH 17/27] flipped service_state in ZoneManager and fixed tests --- nova/scheduler/api.py | 8 +++----- nova/scheduler/zone_manager.py | 20 +++++++++----------- nova/tests/test_zones.py | 18 ++++++------------ 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 6bb3bf3c..816ae551 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -76,11 +76,9 @@ def zone_update(context, zone_id, data): return db.zone_update(context, zone_id, data) -def get_zone_capabilities(context, service=None): - """Returns a dict of key, value capabilities for this zone, - or for a particular class of services running in this zone.""" - return _call_scheduler('get_zone_capabilities', context=context, - params=dict(service=service)) +def get_zone_capabilities(context): + """Returns a dict of key, value capabilities for this zone.""" + return _call_scheduler('get_zone_capabilities', context=context) def update_service_capabilities(context, service_name, host, capabilities): diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 198f9d4c..3ddf6f3c 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -106,28 +106,26 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} # { : ZoneState } - self.service_states = {} # { : { : { cap k : v }}} + self.service_states = {} # { : { : { cap k : v }}} self.green_pool = greenpool.GreenPool() def get_zone_list(self): """Return the list of zones we know about.""" return [zone.to_dict() for zone in self.zone_states.values()] - def get_zone_capabilities(self, context, service=None): + def get_zone_capabilities(self, context): """Roll up all the individual host info to generic 'service' capabilities. Each capability is aggregated into _min and _max values.""" - service_dict = self.service_states - if service: - service_dict = {service: self.service_states.get(service, {})} + hosts_dict = self.service_states # TODO(sandy) - be smarter about fabricating this structure. # But it's likely to change once we understand what the Best-Match # code will need better. combined = {} # { _ : (min, max), ... } - for service_name, host_dict in service_dict.iteritems(): - for host, caps_dict in host_dict.iteritems(): - for cap, value in caps_dict.iteritems(): + for host, host_dict in hosts_dict.iteritems(): + for service_name, service_dict in host_dict.iteritems(): + for cap, value in service_dict.iteritems(): key = "%s_%s" % (service_name, cap) min_value, max_value = combined.get(key, (value, value)) min_value = min(min_value, value) @@ -171,6 +169,6 @@ class ZoneManager(object): """Update the per-service capabilities based on this notification.""" logging.debug(_("Received %(service_name)s service update from " "%(host)s: %(capabilities)s") % locals()) - service_caps = self.service_states.get(service_name, {}) - service_caps[host] = capabilities - self.service_states[service_name] = service_caps + service_caps = self.service_states.get(host, {}) + service_caps[service_name] = capabilities + self.service_states[host] = service_caps diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 688dc704..e132809d 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -78,38 +78,32 @@ class ZoneManagerTestCase(test.TestCase): def test_service_capabilities(self): zm = zone_manager.ZoneManager() - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, {}) zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(1, 1), svc1_b=(2, 2))) zm.update_service_capabilities("svc1", "host1", dict(a=2, b=3)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 2), svc1_b=(3, 3))) zm.update_service_capabilities("svc1", "host2", dict(a=20, b=30)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30))) zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc10_a=(99, 99), svc10_b=(99, 99))) zm.update_service_capabilities("svc1", "host3", dict(c=5)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc1_c=(5, 5), svc10_a=(99, 99), svc10_b=(99, 99))) - caps = zm.get_zone_capabilities(self, 'svc1') - self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), - svc1_c=(5, 5))) - caps = zm.get_zone_capabilities(self, 'svc10') - self.assertEquals(caps, dict(svc10_a=(99, 99), svc10_b=(99, 99))) - def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() From 314541ae34290651b6a9020e60f47d0adf30e0d8 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 08:13:22 -0700 Subject: [PATCH 18/27] print statements removed --- nova/scheduler/query.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 4332f43b..1a6705b4 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -236,7 +236,6 @@ class JsonQuery: if arg != None: cooked_args.append(arg) result = method(self, cooked_args) - print "*** %s %s = %s" % (cmd, cooked_args, result) return result def filter_hosts(self, zone_manager, query): @@ -244,7 +243,6 @@ class JsonQuery: expanded = json.loads(query) hosts = [] for host, services in zone_manager.service_states.iteritems(): - print "-----" r = self._process_query(zone_manager, expanded, host, services) if isinstance(r, list): r = True in r From eb2efc7b40d891470b005f5144b3f90e539f3956 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 10:50:59 -0700 Subject: [PATCH 19/27] merge prop fixes --- nova/scheduler/query.py | 37 ++++++++++++++++++++++------------ nova/tests/test_query.py | 43 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1a6705b4..1e294b59 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -40,19 +40,23 @@ flags.DEFINE_string('default_query_engine', 'Which query engine to use for filtering hosts.') -class Query: +class Query(object): """Base class for query plug-ins.""" def instance_type_to_query(self, instance_type): """Convert instance_type into a query for most common use-case.""" - raise exception.BadSchedulerQueryDriver() + raise NotImplementedError() def filter_hosts(self, zone_manager, query): """Return a list of hosts that fulfill the query.""" - raise exception.BadSchedulerQueryDriver() + raise NotImplementedError() + + def _full_name(self): + """module.classname of the Query object""" + return "%s.%s" % (self.__module__, self.__class__.__name__) -class AllHostsQuery: +class AllHostsQuery(Query): """NOP query plug-in. Returns all hosts in ZoneManager. This essentially does what the old Scheduler+Chance used to give us.""" @@ -60,7 +64,7 @@ class AllHostsQuery: def instance_type_to_query(self, instance_type): """Return anything to prevent base-class from raising exception.""" - return (str(self.__class__), instance_type) + return (self._full_name(), instance_type) def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" @@ -68,12 +72,12 @@ class AllHostsQuery: for host, services in zone_manager.service_states.iteritems()] -class FlavorQuery: +class FlavorQuery(Query): """Query plug-in hard-coded to work with flavors.""" def instance_type_to_query(self, instance_type): """Use instance_type to filter hosts.""" - return (str(self.__class__), instance_type) + return (self._full_name(), instance_type) def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" @@ -115,7 +119,7 @@ class FlavorQuery: #rxtx_cap = Column(Integer, nullable=False, default=0) -class JsonQuery: +class JsonQuery(Query): """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" def _equals(self, args): @@ -175,14 +179,17 @@ class JsonQuery: return True def _not(self, args): + """Flip each of the arguments.""" if len(args) == 0: return False return [not arg for arg in args] def _or(self, args): + """True if any arg is True.""" return True in args def _and(self, args): + """True if all args are True.""" return False not in args commands = { @@ -205,7 +212,7 @@ class JsonQuery: ['>=', '$compute.host_memory.free', required_ram], ['>=', '$compute.disk.available', required_disk] ] - return (str(self.__class__), json.dumps(query)) + return (self._full_name(), json.dumps(query)) def _parse_string(self, string, host, services): """Strings prefixed with $ are capability lookups in the @@ -223,6 +230,7 @@ class JsonQuery: return services def _process_query(self, zone_manager, query, host, services): + """Recursively parse the query structure.""" if len(query) == 0: return True cmd = query[0] @@ -251,15 +259,18 @@ class JsonQuery: return hosts -# Since the caller may specify which driver to use we need -# to have an authoritative list of what is permissible. DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] def choose_driver(driver_name=None): + """Since the caller may specify which driver to use we need + to have an authoritative list of what is permissible. This + function checks the driver name against a predefined set + of acceptable drivers.""" + if not driver_name: driver_name = FLAGS.default_query_engine for driver in DRIVERS: - if str(driver) == driver_name: - return driver + if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: + return driver() raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 8b894a4e..9497a8c9 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -80,10 +80,12 @@ class QueryTestCase(test.TestCase): def test_choose_driver(self): # Test default driver ... driver = query.choose_driver() - self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + self.assertEquals(driver._full_name(), + 'nova.scheduler.query.AllHostsQuery') # Test valid driver ... driver = query.choose_driver('nova.scheduler.query.FlavorQuery') - self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + self.assertEquals(driver._full_name(), + 'nova.scheduler.query.FlavorQuery') # Test invalid driver ... try: query.choose_driver('does not exist') @@ -165,3 +167,40 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([2, 4, 6, 8, 10], just_hosts): self.assertEquals('host%02d' % index, host) + + # Try some bogus input ... + raw = ['unknown command', ] + cooked = json.dumps(raw) + try: + driver.filter_hosts(self.zone_manager, cooked) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( + ['not', True, False, True, False] + ))) + + try: + driver.filter_hosts(self.zone_manager, json.dumps( + 'not', True, False, True, False + )) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$foo', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$.....', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]] + ))) + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', {}, ['>', '$missing....foo']] + ))) From fd94dbb2d54a8ec745fd4044dd1560d9486ecf69 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 18:09:11 -0700 Subject: [PATCH 20/27] terminology: no more plug-ins or queries. They are host filters and drivers. --- nova/scheduler/{query.py => host_filter.py} | 76 +++++++++++-------- .../{test_query.py => test_host_filter.py} | 44 ++++++----- 2 files changed, 66 insertions(+), 54 deletions(-) rename nova/scheduler/{query.py => host_filter.py} (76%) rename nova/tests/{test_query.py => test_host_filter.py} (83%) diff --git a/nova/scheduler/query.py b/nova/scheduler/host_filter.py similarity index 76% rename from nova/scheduler/query.py rename to nova/scheduler/host_filter.py index 1e294b59..aa6101c9 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/host_filter.py @@ -14,14 +14,23 @@ # under the License. """ -Query is a plug-in mechanism for requesting instance resources. -Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just +Host Filter is a driver mechanism for requesting instance resources. +Three drivers are included: AllHosts, Flavor & JSON. AllHosts just returns the full, unfiltered list of hosts. Flavor is a hard coded matching mechanism based on flavor criteria and JSON is an ad-hoc -query grammar. +filter grammar. -Note: These are hard filters. All capabilities used must be present -or the host will excluded. If you want soft filters use the weighting +Why JSON? The requests for instances may come in through the +REST interface from a user or a parent Zone. +Currently Flavors and/or InstanceTypes are used for +specifing the type of instance desired. Specific Nova users have +noted a need for a more expressive way of specifying instances. +Since we don't want to get into building full DSL this is a simple +form as an example of how this could be done. In reality, most +consumers will use the more rigid filters such as FlavorFilter. + +Note: These are hard filters. All capabilities used must be present +or the host will be excluded. If you want soft filters use the weighting mechanism which is intended for the more touchy-feely capabilities. """ @@ -32,36 +41,36 @@ from nova import flags from nova import log as logging from nova import utils -LOG = logging.getLogger('nova.scheduler.query') +LOG = logging.getLogger('nova.scheduler.host_filter') FLAGS = flags.FLAGS -flags.DEFINE_string('default_query_engine', - 'nova.scheduler.query.AllHostsQuery', - 'Which query engine to use for filtering hosts.') +flags.DEFINE_string('default_host_filter_driver', + 'nova.scheduler.host_filter.AllHostsFilter', + 'Which driver to use for filtering hosts.') -class Query(object): - """Base class for query plug-ins.""" +class HostFilter(object): + """Base class for host filter drivers.""" - def instance_type_to_query(self, instance_type): - """Convert instance_type into a query for most common use-case.""" + def instance_type_to_filter(self, instance_type): + """Convert instance_type into a filter for most common use-case.""" raise NotImplementedError() def filter_hosts(self, zone_manager, query): - """Return a list of hosts that fulfill the query.""" + """Return a list of hosts that fulfill the filter.""" raise NotImplementedError() def _full_name(self): - """module.classname of the Query object""" + """module.classname of the filter driver""" return "%s.%s" % (self.__module__, self.__class__.__name__) -class AllHostsQuery(Query): - """NOP query plug-in. Returns all hosts in ZoneManager. +class AllHostsFilter(HostFilter): + """NOP host filter driver. Returns all hosts in ZoneManager. This essentially does what the old Scheduler+Chance used to give us.""" - def instance_type_to_query(self, instance_type): + def instance_type_to_filter(self, instance_type): """Return anything to prevent base-class from raising exception.""" return (self._full_name(), instance_type) @@ -72,10 +81,10 @@ class AllHostsQuery(Query): for host, services in zone_manager.service_states.iteritems()] -class FlavorQuery(Query): - """Query plug-in hard-coded to work with flavors.""" +class FlavorFilter(HostFilter): + """HostFilter driver hard-coded to work with flavors.""" - def instance_type_to_query(self, instance_type): + def instance_type_to_filter(self, instance_type): """Use instance_type to filter hosts.""" return (self._full_name(), instance_type) @@ -119,8 +128,9 @@ class FlavorQuery(Query): #rxtx_cap = Column(Integer, nullable=False, default=0) -class JsonQuery(Query): - """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" +class JsonFilter(HostFilter): + """Host Filter driver to allow simple JSON-based grammar for + selecting hosts.""" def _equals(self, args): """First term is == all the other terms.""" @@ -204,8 +214,8 @@ class JsonQuery(Query): 'and': _and, } - def instance_type_to_query(self, instance_type): - """Convert instance_type into JSON query object.""" + def instance_type_to_filter(self, instance_type): + """Convert instance_type into JSON filter object.""" required_ram = instance_type['memory_mb'] required_disk = instance_type['local_gb'] query = ['and', @@ -229,7 +239,7 @@ class JsonQuery(Query): return None return services - def _process_query(self, zone_manager, query, host, services): + def _process_filter(self, zone_manager, query, host, services): """Recursively parse the query structure.""" if len(query) == 0: return True @@ -238,7 +248,7 @@ class JsonQuery(Query): cooked_args = [] for arg in query[1:]: if isinstance(arg, list): - arg = self._process_query(zone_manager, arg, host, services) + arg = self._process_filter(zone_manager, arg, host, services) elif isinstance(arg, basestring): arg = self._parse_string(arg, host, services) if arg != None: @@ -247,11 +257,11 @@ class JsonQuery(Query): return result def filter_hosts(self, zone_manager, query): - """Return a list of hosts that can fulfill query.""" + """Return a list of hosts that can fulfill filter.""" expanded = json.loads(query) hosts = [] for host, services in zone_manager.service_states.iteritems(): - r = self._process_query(zone_manager, expanded, host, services) + r = self._process_filter(zone_manager, expanded, host, services) if isinstance(r, list): r = True in r if r: @@ -259,7 +269,7 @@ class JsonQuery(Query): return hosts -DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] +DRIVERS = [AllHostsFilter, FlavorFilter, JsonFilter] def choose_driver(driver_name=None): @@ -267,10 +277,10 @@ def choose_driver(driver_name=None): to have an authoritative list of what is permissible. This function checks the driver name against a predefined set of acceptable drivers.""" - + if not driver_name: - driver_name = FLAGS.default_query_engine + driver_name = FLAGS.default_host_filter_driver for driver in DRIVERS: if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: return driver() - raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) + raise exception.SchedulerHostFilterDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_query.py b/nova/tests/test_host_filter.py similarity index 83% rename from nova/tests/test_query.py rename to nova/tests/test_host_filter.py index 9497a8c9..31e40ae1 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_host_filter.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. """ -Tests For Scheduler Query Drivers +Tests For Scheduler Host Filter Drivers. """ import json @@ -21,7 +21,7 @@ import json from nova import exception from nova import flags from nova import test -from nova.scheduler import query +from nova.scheduler import host_filter FLAGS = flags.FLAGS @@ -30,8 +30,8 @@ class FakeZoneManager: pass -class QueryTestCase(test.TestCase): - """Test case for query drivers.""" +class HostFilterTestCase(test.TestCase): + """Test case for host filter drivers.""" def _host_caps(self, multiplier): # Returns host capabilities in the following way: @@ -57,8 +57,9 @@ class QueryTestCase(test.TestCase): 'host_name-label': 'xs-%s' % multiplier} def setUp(self): - self.old_flag = FLAGS.default_query_engine - FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' + self.old_flag = FLAGS.default_host_filter_driver + FLAGS.default_host_filter_driver = \ + 'nova.scheduler.host_filter.AllHostsFilter' self.instance_type = dict(name='tiny', memory_mb=50, vcpus=10, @@ -75,37 +76,38 @@ class QueryTestCase(test.TestCase): self.zone_manager.service_states = states def tearDown(self): - FLAGS.default_query_engine = self.old_flag + FLAGS.default_host_filter_driver = self.old_flag def test_choose_driver(self): # Test default driver ... - driver = query.choose_driver() + driver = host_filter.choose_driver() self.assertEquals(driver._full_name(), - 'nova.scheduler.query.AllHostsQuery') + 'nova.scheduler.host_filter.AllHostsFilter') # Test valid driver ... - driver = query.choose_driver('nova.scheduler.query.FlavorQuery') + driver = host_filter.choose_driver( + 'nova.scheduler.host_filter.FlavorFilter') self.assertEquals(driver._full_name(), - 'nova.scheduler.query.FlavorQuery') + 'nova.scheduler.host_filter.FlavorFilter') # Test invalid driver ... try: - query.choose_driver('does not exist') + host_filter.choose_driver('does not exist') self.fail("Should not find driver") - except exception.SchedulerQueryDriverNotFound: + except exception.SchedulerHostFilterDriverNotFound: pass def test_all_host_driver(self): - driver = query.AllHostsQuery() - cooked = driver.instance_type_to_query(self.instance_type) + driver = host_filter.AllHostsFilter() + cooked = driver.instance_type_to_filter(self.instance_type) hosts = driver.filter_hosts(self.zone_manager, cooked) self.assertEquals(10, len(hosts)) for host, capabilities in hosts: self.assertTrue(host.startswith('host')) def test_flavor_driver(self): - driver = query.FlavorQuery() + driver = host_filter.FlavorFilter() # filter all hosts that can support 50 ram and 500 disk - name, cooked = driver.instance_type_to_query(self.instance_type) - self.assertEquals('nova.scheduler.query.FlavorQuery', name) + name, cooked = driver.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.FlavorFilter', name) hosts = driver.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -114,10 +116,10 @@ class QueryTestCase(test.TestCase): self.assertEquals('host10', just_hosts[5]) def test_json_driver(self): - driver = query.JsonQuery() + driver = host_filter.JsonFilter() # filter all hosts that can support 50 ram and 500 disk - name, cooked = driver.instance_type_to_query(self.instance_type) - self.assertEquals('nova.scheduler.query.JsonQuery', name) + name, cooked = driver.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) hosts = driver.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] From 4a89cfc4898cd189ee65a13787d124bbf67accfc Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 6 May 2011 09:06:46 -0700 Subject: [PATCH 21/27] revised file docs --- nova/scheduler/host_filter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index aa6101c9..ed8e65c7 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -29,9 +29,11 @@ Since we don't want to get into building full DSL this is a simple form as an example of how this could be done. In reality, most consumers will use the more rigid filters such as FlavorFilter. -Note: These are hard filters. All capabilities used must be present -or the host will be excluded. If you want soft filters use the weighting -mechanism which is intended for the more touchy-feely capabilities. +Note: These are "required" capability filters. These capabilities +used must be present or the host will be excluded. The hosts +returned are then weighed by the Weighted Scheduler. Weights +can take the more esoteric factors into consideration (such as +server affinity and customer separation). """ import json From 96cb4fae342cf4ef618e85bcb3a5070efd9df71e Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 6 May 2011 11:04:00 -0700 Subject: [PATCH 22/27] tests pass again --- nova/tests/test_compute.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 39311079..55e7ae0c 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -21,6 +21,7 @@ Tests For Compute import datetime import mox +import stubout from nova import compute from nova import context @@ -52,6 +53,10 @@ class FakeTime(object): self.counter += t +def nop_report_driver_status(self): + pass + + class ComputeTestCase(test.TestCase): """Test case for compute""" def setUp(self): @@ -649,6 +654,10 @@ class ComputeTestCase(test.TestCase): def test_run_kill_vm(self): """Detect when a vm is terminated behind the scenes""" + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(compute_manager.ComputeManager, + '_report_driver_status', nop_report_driver_status) + instance_id = self._create_instance() self.compute.run_instance(self.context, instance_id) From dc3fcffb182ef659b08503af19f640de6f3a571b Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 9 May 2011 08:23:25 -0700 Subject: [PATCH 23/27] basic test working --- nova/tests/test_xenapi.py | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 375480a2..756a289b 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -17,6 +17,7 @@ """Test suite for XenAPI.""" import functools +import json import os import re import stubout @@ -665,3 +666,42 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_VHD self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_VHD) + + +class FakeXenApi(object): + """Fake XenApi for testing HostState.""" + + class FakeSR(object): + def get_record(self, ref): + return {'virtual_allocation':10000, + 'physical_utilisation':20000} + + SR = FakeSR() + + +class FakeSession(object): + """Fake Session class for HostState testing.""" + + def async_call_plugin(self, *args): + return None + + def wait_for_task(self, *args): + return json.dumps({}) + + def get_xenapi(self): + return FakeXenApi() + + +class HostStateTestCase(test.TestCase): + """Tests HostState, which holds metrics from XenServer that get + reported back to the Schedulers.""" + + def _fake_safe_find_sr(self, session): + """None SR ref since we're ignoring it in FakeSR.""" + return None + + def test_host_state(self): + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(vm_utils, 'safe_find_sr', self._fake_safe_find_sr) + host_state = xenapi_conn.HostState(FakeSession()) + From cf65ae85a9ba5eb410987c6b57772c0d61013e4e Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 9 May 2011 09:08:56 -0700 Subject: [PATCH 24/27] capabilities flattened and tests fixed --- nova/scheduler/host_filter.py | 16 ++++++++-------- nova/tests/test_host_filter.py | 26 +++++++++++++------------- nova/tests/test_xenapi.py | 12 ++++++------ 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 3e831b76..885878e1 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -96,8 +96,8 @@ class FlavorFilter(HostFilter): selected_hosts = [] for host, services in zone_manager.service_states.iteritems(): capabilities = services.get('compute', {}) - host_ram_mb = capabilities['host_memory']['free'] - disk_bytes = capabilities['disk']['available'] + host_ram_mb = capabilities['host_memory_free'] + disk_bytes = capabilities['disk_available'] if host_ram_mb >= instance_type['memory_mb'] and \ disk_bytes >= instance_type['local_gb']: selected_hosts.append((host, capabilities)) @@ -106,10 +106,10 @@ class FlavorFilter(HostFilter): #host entries (currently) are like: # {'host_name-description': 'Default install of XenServer', # 'host_hostname': 'xs-mini', -# 'host_memory': {'total': 8244539392, -# 'overhead': 184225792, -# 'free': 3868327936, -# 'free-computed': 3840843776}, +# 'host_memory_total': 8244539392, +# 'host_memory_overhead': 184225792, +# 'host_memory_free': 3868327936, +# 'host_memory_free-computed': 3840843776}, # 'host_other-config': {}, # 'host_ip_address': '192.168.1.109', # 'host_cpu_info': {}, @@ -221,8 +221,8 @@ class JsonFilter(HostFilter): required_ram = instance_type['memory_mb'] required_disk = instance_type['local_gb'] query = ['and', - ['>=', '$compute.host_memory.free', required_ram], - ['>=', '$compute.disk.available', required_disk] + ['>=', '$compute.host_memory_free', required_ram], + ['>=', '$compute.disk_available', required_disk] ] return (self._full_name(), json.dumps(query)) diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py index 31e40ae1..c029d41e 100644 --- a/nova/tests/test_host_filter.py +++ b/nova/tests/test_host_filter.py @@ -43,16 +43,16 @@ class HostFilterTestCase(test.TestCase): # which means ... don't go above 10 hosts. return {'host_name-description': 'XenServer %s' % multiplier, 'host_hostname': 'xs-%s' % multiplier, - 'host_memory': {'total': 100, - 'overhead': 10, - 'free': 10 + multiplier * 10, - 'free-computed': 10 + multiplier * 10}, + 'host_memory_total': 100, + 'host_memory_overhead': 10, + 'host_memory_free': 10 + multiplier * 10, + 'host_memory_free-computed': 10 + multiplier * 10, 'host_other-config': {}, 'host_ip_address': '192.168.1.%d' % (100 + multiplier), 'host_cpu_info': {}, - 'disk': {'available': 100 + multiplier * 100, - 'total': 1000, - 'used': 0}, + 'disk_available': 100 + multiplier * 100, + 'disk_total': 1000, + 'disk_used': 0, 'host_uuid': 'xxx-%d' % multiplier, 'host_name-label': 'xs-%s' % multiplier} @@ -131,12 +131,12 @@ class HostFilterTestCase(test.TestCase): raw = ['or', ['and', - ['<', '$compute.host_memory.free', 30], - ['<', '$compute.disk.available', 300] + ['<', '$compute.host_memory_free', 30], + ['<', '$compute.disk_available', 300] ], ['and', - ['>', '$compute.host_memory.free', 70], - ['>', '$compute.disk.available', 700] + ['>', '$compute.host_memory_free', 70], + ['>', '$compute.disk_available', 700] ] ] cooked = json.dumps(raw) @@ -149,7 +149,7 @@ class HostFilterTestCase(test.TestCase): self.assertEquals('host%02d' % index, host) raw = ['not', - ['=', '$compute.host_memory.free', 30], + ['=', '$compute.host_memory_free', 30], ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -160,7 +160,7 @@ class HostFilterTestCase(test.TestCase): for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] + raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 0f1b2aa4..67829157 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -709,9 +709,9 @@ class HostStateTestCase(test.TestCase): self.stubs.Set(vm_utils, 'safe_find_sr', self._fake_safe_find_sr) host_state = xenapi_conn.HostState(FakeSession()) stats = host_state._stats - self.assertEquals('disk_total', 10000) - self.assertEquals('disk_used', 20000) - self.assertEquals('host_memory_total', 10) - self.assertEquals('host_memory_overhead', 20) - self.assertEquals('host_memory_free', 30) - self.assertEquals('host_memory_free-computed', 40) + self.assertEquals(stats['disk_total'], 10000) + self.assertEquals(stats['disk_used'], 20000) + self.assertEquals(stats['host_memory_total'], 10) + self.assertEquals(stats['host_memory_overhead'], 20) + self.assertEquals(stats['host_memory_free'], 30) + self.assertEquals(stats['host_memory_free-computed'], 40) From b1aa6e3c712dc84b58de21d1b823d2ca6c8815be Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 9 May 2011 09:10:22 -0700 Subject: [PATCH 25/27] pep8 --- nova/tests/test_xenapi.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 67829157..6dbd1aee 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -673,24 +673,24 @@ class FakeXenApi(object): class FakeSR(object): def get_record(self, ref): - return {'virtual_allocation':10000, - 'physical_utilisation':20000} + return {'virtual_allocation': 10000, + 'physical_utilisation': 20000} SR = FakeSR() class FakeSession(object): """Fake Session class for HostState testing.""" - + def async_call_plugin(self, *args): return None def wait_for_task(self, *args): - vm = {'total':10, - 'overhead':20, - 'free':30, - 'free-computed':40} - return json.dumps({'host_memory':vm}) + vm = {'total': 10, + 'overhead': 20, + 'free': 30, + 'free-computed': 40} + return json.dumps({'host_memory': vm}) def get_xenapi(self): return FakeXenApi() From f25040fe825a3c28831a4a27b78ced02ccaf44cc Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 9 May 2011 12:57:56 -0700 Subject: [PATCH 26/27] unified underscore/dash issue --- nova/scheduler/host_filter.py | 2 +- nova/tests/test_xenapi.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 885878e1..483f3225 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -109,7 +109,7 @@ class FlavorFilter(HostFilter): # 'host_memory_total': 8244539392, # 'host_memory_overhead': 184225792, # 'host_memory_free': 3868327936, -# 'host_memory_free-computed': 3840843776}, +# 'host_memory_free_computed': 3840843776}, # 'host_other-config': {}, # 'host_ip_address': '192.168.1.109', # 'host_cpu_info': {}, diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 6dbd1aee..6072f545 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -714,4 +714,4 @@ class HostStateTestCase(test.TestCase): self.assertEquals(stats['host_memory_total'], 10) self.assertEquals(stats['host_memory_overhead'], 20) self.assertEquals(stats['host_memory_free'], 30) - self.assertEquals(stats['host_memory_free-computed'], 40) + self.assertEquals(stats['host_memory_free_computed'], 40) From 50acab28f279c6266711cf4ef81b66ad88c48d62 Mon Sep 17 00:00:00 2001 From: Justin Shepherd Date: Mon, 9 May 2011 22:36:01 -0500 Subject: [PATCH 27/27] Added GitPython to [install_dir]/tools/pip-requires. --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 60e1d2da..d7f70f41 100644 --- a/Authors +++ b/Authors @@ -44,6 +44,7 @@ Josh Kearney Josh Kleinpeter Joshua McKenty Justin Santa Barbara +Justin Shepherd Kei Masumoto Ken Pepple Kevin Bringard