From bfff03f74c2b4195986bf0c0d30250b97e11dbad Mon Sep 17 00:00:00 2001 From: John Tran Date: Fri, 18 Mar 2011 11:49:11 -0700 Subject: [PATCH 01/24] created api endpoint to allow uploading of public key --- nova/tests/test_cloud.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index cf8ee7ef..03b1ad2f 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -279,6 +279,22 @@ class CloudTestCase(test.TestCase): self.assertTrue(filter(lambda k: k['keyName'] == 'test1', keys)) self.assertTrue(filter(lambda k: k['keyName'] == 'test2', keys)) + def test_import_public_key(self): + result = self.cloud.import_public_key(self.context, + 'testimportkey', 'mytestpubkey', 'mytestfprint') + self.assertTrue(result) + keydata = db.key_pair_get(self.context, + self.context.user.id, + 'testimportkey') + print "PUBLIC_KEY:" + file = open('/tmp/blah', 'w') + file.write(keydata['public_key']) + file.close() + print keydata['public_key'] + self.assertEqual('mytestpubkey', keydata['public_key']) + self.assertEqual('mytestfprint', keydata['fingerprint']) + self.assertTrue(1) + def test_delete_key_pair(self): self._create_key('test') self.cloud.delete_key_pair(self.context, 'test') From 4c8178a75a74e5c327a06824e1c7792f2e9ff0e1 Mon Sep 17 00:00:00 2001 From: John Tran Date: Fri, 18 Mar 2011 12:17:40 -0700 Subject: [PATCH 02/24] cleaned up tests stubs that were accidentally checked in --- nova/tests/test_cloud.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 03b1ad2f..3a266c99 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -281,19 +281,15 @@ class CloudTestCase(test.TestCase): def test_import_public_key(self): result = self.cloud.import_public_key(self.context, - 'testimportkey', 'mytestpubkey', 'mytestfprint') + 'testimportkey', + 'mytestpubkey', + 'mytestfprint') self.assertTrue(result) keydata = db.key_pair_get(self.context, self.context.user.id, 'testimportkey') - print "PUBLIC_KEY:" - file = open('/tmp/blah', 'w') - file.write(keydata['public_key']) - file.close() - print keydata['public_key'] self.assertEqual('mytestpubkey', keydata['public_key']) self.assertEqual('mytestfprint', keydata['fingerprint']) - self.assertTrue(1) def test_delete_key_pair(self): self._create_key('test') From 0f22d5871d519a649455d3de642ad719b443f978 Mon Sep 17 00:00:00 2001 From: John Tran Date: Mon, 21 Mar 2011 14:35:19 -0700 Subject: [PATCH 03/24] if fingerprint data not provided, added logic to calculate it using the pub key. --- nova/tests/public_key/dummy.fingerprint | 1 + nova/tests/public_key/dummy.pub | 1 + nova/tests/test_cloud.py | 32 +++++++++++++++++++------ 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 nova/tests/public_key/dummy.fingerprint create mode 100644 nova/tests/public_key/dummy.pub diff --git a/nova/tests/public_key/dummy.fingerprint b/nova/tests/public_key/dummy.fingerprint new file mode 100644 index 00000000..715bca27 --- /dev/null +++ b/nova/tests/public_key/dummy.fingerprint @@ -0,0 +1 @@ +1c:87:d1:d9:32:fd:62:3c:78:2b:c0:ad:c0:15:88:df diff --git a/nova/tests/public_key/dummy.pub b/nova/tests/public_key/dummy.pub new file mode 100644 index 00000000..d4cf2bc0 --- /dev/null +++ b/nova/tests/public_key/dummy.pub @@ -0,0 +1 @@ +ssh-dss AAAAB3NzaC1kc3MAAACBAMGJlY9XEIm2X234pdO5yFWMp2JuOQx8U0E815IVXhmKxYCBK9ZakgZOIQmPbXoGYyV+mziDPp6HJ0wKYLQxkwLEFr51fAZjWQvRss0SinURRuLkockDfGFtD4pYJthekr/rlqMKlBSDUSpGq8jUWW60UJ18FGooFpxR7ESqQRx/AAAAFQC96LRglaUeeP+E8U/yblEJocuiWwAAAIA3XiMR8Skiz/0aBm5K50SeQznQuMJTyzt9S9uaz5QZWiFu69hOyGSFGw8fqgxEkXFJIuHobQQpGYQubLW0NdaYRqyE/Vud3JUJUb8Texld6dz8vGemyB5d1YvtSeHIo8/BGv2msOqR3u5AZTaGCBD9DhpSGOKHEdNjTtvpPd8S8gAAAIBociGZ5jf09iHLVENhyXujJbxfGRPsyNTyARJfCOGl0oFV6hEzcQyw8U/ePwjgvjc2UizMWLl8tsb2FXKHRdc2v+ND3Us+XqKQ33X3ADP4FZ/+Oj213gMyhCmvFTP0u5FmHog9My4CB7YcIWRuUR42WlhQ2IfPvKwUoTk3R+T6Og== www-data@mk diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 3a266c99..c49a39ed 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -280,16 +280,34 @@ class CloudTestCase(test.TestCase): self.assertTrue(filter(lambda k: k['keyName'] == 'test2', keys)) def test_import_public_key(self): - result = self.cloud.import_public_key(self.context, - 'testimportkey', - 'mytestpubkey', - 'mytestfprint') - self.assertTrue(result) + # test when user provides all values + result1 = self.cloud.import_public_key(self.context, + 'testimportkey1', + 'mytestpubkey', + 'mytestfprint') + self.assertTrue(result1) keydata = db.key_pair_get(self.context, - self.context.user.id, - 'testimportkey') + self.context.user.id, + 'testimportkey1') self.assertEqual('mytestpubkey', keydata['public_key']) self.assertEqual('mytestfprint', keydata['fingerprint']) + # test when user omits fingerprint + pubkey_path = os.path.join(os.path.dirname(__file__), 'public_key') + f = open(pubkey_path + '/dummy.pub', 'r') + dummypub = f.readline().rstrip() + f.close + f = open(pubkey_path + '/dummy.fingerprint', 'r') + dummyfprint = f.readline().rstrip() + f.close + result2 = self.cloud.import_public_key(self.context, + 'testimportkey2', + dummypub) + self.assertTrue(result2) + keydata = db.key_pair_get(self.context, + self.context.user.id, + 'testimportkey2') + self.assertEqual(dummypub, keydata['public_key']) + self.assertEqual(dummyfprint, keydata['fingerprint']) def test_delete_key_pair(self): self._create_key('test') From b3b03a75e02f015062de2ff9ad0c7f2ad42e5b6c Mon Sep 17 00:00:00 2001 From: John Tran Date: Wed, 23 Mar 2011 11:16:22 -0700 Subject: [PATCH 04/24] added myself to authors file --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 7993955e..c1e16489 100644 --- a/Authors +++ b/Authors @@ -28,6 +28,7 @@ Jesse Andrews Joe Heck Joel Moore John Dewey +John Tran Jonathan Bryce Jordan Rinke Josh Durgin From d924c7392021c7d3bff038b8b0163fc8e4175fd2 Mon Sep 17 00:00:00 2001 From: Renuka Apte Date: Tue, 12 Apr 2011 15:20:30 -0700 Subject: [PATCH 05/24] Minor fixes --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index eccf38a4..b6da7a43 100644 --- a/Authors +++ b/Authors @@ -56,6 +56,7 @@ Nachi Ueno Naveed Massjouni Nirmal Ranganathan Paul Voccio +Renuka Apte Ricardo Carrillo Cruz Rick Clark Rick Harris From 74aff1dbd29bd872d95791300e83bb73f19257be Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 11 May 2011 06:28:07 -0700 Subject: [PATCH 06/24] First cut with tests passing --- nova/scheduler/api.py | 6 ++ nova/scheduler/zone_aware_scheduler.py | 88 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 nova/scheduler/zone_aware_scheduler.py diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 816ae551..d8a0025e 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -81,6 +81,12 @@ def get_zone_capabilities(context): return _call_scheduler('get_zone_capabilities', context=context) +def select(context, specs=None): + """Returns a list of hosts.""" + return _call_scheduler('select', context=context, + params={"specs": specs}) + + def update_service_capabilities(context, service_name, host, capabilities): """Send an update to all the scheduler services informing them of the capabilities of this service.""" diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py new file mode 100644 index 00000000..b849e8de --- /dev/null +++ b/nova/scheduler/zone_aware_scheduler.py @@ -0,0 +1,88 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +The Zone Aware Scheduler is a base class Scheduler for creating instances +across zones. There are two expansion points to this class for: +1. Assigning Weights to hosts for requested instances +2. Filtering Hosts based on required instance capabilities +""" + +import operator + +from nova import log as logging +from nova.scheduler import api + +LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler') + + +class ZoneAwareScheduler(object): + """Base class for creating Zone Aware Schedulers.""" + + def _call_zone_method(self, context, method, specs): + """Call novaclient zone method. Broken out for testing.""" + return api.call_zone_method(context, method, specs=specs) + + def select(self, context, *args, **kwargs): + """Select returns a list of weights and zone/host information + corresponding to the best hosts to service the request. Any + child zone information has been encrypted so as not to reveal + anything about the children.""" + return self._schedule(context, "compute", *args, **kwargs) + + def schedule(self, context, topic, *args, **kwargs): + """The schedule() contract requires we return the one + best-suited host for this request. + """ + res = self._schedule(context, topic, *args, **kwargs) + return res[0] + + def _schedule(self, context, topic, *args, **kwargs): + """Returns a list of hosts that meet the required specs, + ordered by their fitness. + """ + # Filter local hosts based on requirements ... + host_list = self.filter_hosts() + + # then weigh the selected hosts. + # weighted = [ { 'weight':#, 'name':host, ...}, ] + weighted = self.weight_hosts(host_list) + + # Next, tack on the best weights from the child zones ... + child_results = self._call_zone_method(context, "select", + specs=specs) + for child_zone, result in child_results: + for weighting in result: + # Remember the child_zone so we can get back to + # it later if needed. This implicitly builds a zone + # path structure. + host_dict = { + "weight": weighting["weight"], + "child_zone": child_zone, + "child_blob": weighting["blob"]} + weighted.append(host_dict) + + weighted.sort(key=operator.itemgetter('weight')) + return weighted + + def filter_hosts(self): + """Derived classes must override this method and return + a list of hosts in [?] format.""" + raise NotImplemented() + + def weigh_hosts(self, hosts): + """Derived classes must override this method and return + a lists of hosts in [?] format.""" + raise NotImplemented() From 91b8ac3f980f74ba0de610ce55f7444d12213f90 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 11 May 2011 11:12:31 -0700 Subject: [PATCH 07/24] start of zone_aware_scheduler test --- nova/scheduler/api.py | 39 ++++++++++++++++++++++++++ nova/scheduler/zone_aware_scheduler.py | 31 ++++++++++++-------- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index d8a0025e..55f8e0a6 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -111,6 +111,45 @@ def _process(func, zone): return func(nova, zone) +def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs): + """Returns a list of (zone, call_result) objects.""" + if not isinstance(errors_to_ignore, (list, tuple)): + # This will also handle the default None + errors_to_ignore = [errors_to_ignore] + + pool = greenpool.GreenPool() + results = [] + for zone in db.zone_get_all(context): + try: + nova = novaclient.OpenStack(zone.username, zone.password, + zone.api_url) + nova.authenticate() + except novaclient.exceptions.BadRequest, e: + url = zone.api_url + LOG.warn(_("Failed request to zone; URL=%(url)s: %(e)s") + % locals()) + #TODO (dabo) - add logic for failure counts per zone, + # with escalation after a given number of failures. + continue + zone_method = getattr(nova.zones, method) + + def _error_trap(*args, **kwargs): + try: + return zone_method(*args, **kwargs) + except Exception as e: + if type(e) in errors_to_ignore: + return None + # TODO (dabo) - want to be able to re-raise here. + # Returning a string now; raising was causing issues. + # raise e + return "ERROR", "%s" % e + + res = pool.spawn(_error_trap, *args, **kwargs) + results.append((zone, res)) + pool.waitall() + return [(zone.id, res.wait()) for zone, res in results] + + def child_zone_helper(zone_list, func): """Fire off a command to each zone in the list. The return is [novaclient return objects] from each child zone. diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index b849e8de..b85cdfe6 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -24,11 +24,12 @@ import operator from nova import log as logging from nova.scheduler import api +from nova.scheduler import driver LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler') -class ZoneAwareScheduler(object): +class ZoneAwareScheduler(driver.Scheduler): """Base class for creating Zone Aware Schedulers.""" def _call_zone_method(self, context, method, specs): @@ -42,23 +43,29 @@ class ZoneAwareScheduler(object): anything about the children.""" return self._schedule(context, "compute", *args, **kwargs) - def schedule(self, context, topic, *args, **kwargs): + def schedule(self, context, topic, *args, **kwargs): """The schedule() contract requires we return the one best-suited host for this request. """ res = self._schedule(context, topic, *args, **kwargs) + # TODO(sirp): should this be a host object rather than a weight-dict? return res[0] def _schedule(self, context, topic, *args, **kwargs): """Returns a list of hosts that meet the required specs, ordered by their fitness. """ + + #TODO(sandy): extract these from args. + num_instances = 1 + specs = {} + # Filter local hosts based on requirements ... - host_list = self.filter_hosts() + host_list = self.filter_hosts(num_instances, specs) # then weigh the selected hosts. # weighted = [ { 'weight':#, 'name':host, ...}, ] - weighted = self.weight_hosts(host_list) + weighted = self.weigh_hosts(num_instances, specs, host_list) # Next, tack on the best weights from the child zones ... child_results = self._call_zone_method(context, "select", @@ -77,12 +84,12 @@ class ZoneAwareScheduler(object): weighted.sort(key=operator.itemgetter('weight')) return weighted - def filter_hosts(self): - """Derived classes must override this method and return - a list of hosts in [?] format.""" - raise NotImplemented() + def filter_hosts(self, num, specs): + """Derived classes must override this method and return + a list of hosts in [?] format.""" + raise NotImplemented() - def weigh_hosts(self, hosts): - """Derived classes must override this method and return - a lists of hosts in [?] format.""" - raise NotImplemented() + def weigh_hosts(self, num, specs, hosts): + """Derived classes must override this method and return + a lists of hosts in [?] format.""" + raise NotImplemented() From 0087f66d8aa4ec412265f49c512d2909fb8e5033 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 11 May 2011 11:43:58 -0700 Subject: [PATCH 08/24] NoValidHost exception test --- nova/scheduler/zone_aware_scheduler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index b85cdfe6..8285ec57 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -49,6 +49,8 @@ class ZoneAwareScheduler(driver.Scheduler): """ res = self._schedule(context, topic, *args, **kwargs) # TODO(sirp): should this be a host object rather than a weight-dict? + if not res: + raise driver.NoValidHost(_('No hosts were available')) return res[0] def _schedule(self, context, topic, *args, **kwargs): @@ -64,7 +66,7 @@ class ZoneAwareScheduler(driver.Scheduler): host_list = self.filter_hosts(num_instances, specs) # then weigh the selected hosts. - # weighted = [ { 'weight':#, 'name':host, ...}, ] + # weighted = [{weight=weight, name=hostname}, ...] weighted = self.weigh_hosts(num_instances, specs, host_list) # Next, tack on the best weights from the child zones ... @@ -86,10 +88,10 @@ class ZoneAwareScheduler(driver.Scheduler): def filter_hosts(self, num, specs): """Derived classes must override this method and return - a list of hosts in [?] format.""" + a list of hosts in [(hostname, capability_dict)] format.""" raise NotImplemented() def weigh_hosts(self, num, specs, hosts): """Derived classes must override this method and return - a lists of hosts in [?] format.""" + a lists of hosts in [(weight, hostname)] format.""" raise NotImplemented() From 3e494c104a52f6a848f8ba969de9f80eb32d31e0 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 11 May 2011 12:45:22 -0700 Subject: [PATCH 09/24] messing around with the flow of create() and specs --- nova/scheduler/zone_aware_scheduler.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 8285ec57..07f86450 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -36,6 +36,24 @@ class ZoneAwareScheduler(driver.Scheduler): """Call novaclient zone method. Broken out for testing.""" return api.call_zone_method(context, method, specs=specs) + def schedule_run_instance(self, context, topic='compute', specs=None, + *args, **kwargs): + """This method is called from nova.compute.api to provision + an instance. However we need to look at the parameters being + passed in to see if this is a request to: + 1. Create a Build Plan and then provision, or + 2. Use the Build Plan information in the request parameters + to simply create the instance (either in this zone or + a child zone).""" + + if 'blob' in specs: + return self.provision_instance(context, topic, specs) + + # Create build plan and provision ... + build_plan = self.select(context, specs) + for item in build_plan: + self.provision_instance(context, topic, item) + def select(self, context, *args, **kwargs): """Select returns a list of weights and zone/host information corresponding to the best hosts to service the request. Any From 0f2c1a6f4c79349445871d99bb6dcccdb989d2d1 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Thu, 12 May 2011 20:07:54 -0500 Subject: [PATCH 10/24] Adding basic tests for call_zone_method --- nova/tests/test_scheduler.py | 61 +++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 968ef9d6..54b3f80f 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -912,7 +912,8 @@ class SimpleDriverTestCase(test.TestCase): class FakeZone(object): - def __init__(self, api_url, username, password): + def __init__(self, id, api_url, username, password): + self.id = id self.api_url = api_url self.username = username self.password = password @@ -920,7 +921,7 @@ class FakeZone(object): def zone_get_all(context): return [ - FakeZone('http://example.com', 'bob', 'xxx'), + FakeZone(1, 'http://example.com', 'bob', 'xxx'), ] @@ -1037,7 +1038,7 @@ class FakeNovaClient(object): class DynamicNovaClientTest(test.TestCase): def test_issue_novaclient_command_found(self): - zone = FakeZone('http://example.com', 'bob', 'xxx') + zone = FakeZone(1, 'http://example.com', 'bob', 'xxx') self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeServerCollection()), zone, "servers", "get", 100).a, 10) @@ -1051,7 +1052,7 @@ class DynamicNovaClientTest(test.TestCase): zone, "servers", "pause", 100), None) def test_issue_novaclient_command_not_found(self): - zone = FakeZone('http://example.com', 'bob', 'xxx') + zone = FakeZone(1, 'http://example.com', 'bob', 'xxx') self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), zone, "servers", "get", 100), None) @@ -1063,3 +1064,55 @@ class DynamicNovaClientTest(test.TestCase): self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), zone, "servers", "any", "name"), None) + + +class FakeZonesProxy(object): + def do_something(*args, **kwargs): + return 42 + + def raises_exception(*args, **kwargs): + raise Exception('testing') + + +class FakeNovaClientOpenStack(object): + def __init__(self, *args, **kwargs): + self.zones = FakeZonesProxy() + + def authenticate(self): + pass + + +class CallZoneMethodTest(test.TestCase): + def setUp(self): + super(CallZoneMethodTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(db, 'zone_get_all', zone_get_all) + self.stubs.Set(novaclient, 'OpenStack', FakeNovaClientOpenStack) + + def tearDown(self): + self.stubs.UnsetAll() + super(CallZoneMethodTest, self).tearDown() + + def test_call_zone_method(self): + context = {} + method = 'do_something' + results = api.call_zone_method(context, method) + expected = [(1, 42)] + self.assertEqual(expected, results) + + def test_call_zone_method_not_present(self): + context = {} + method = 'not_present' + self.assertRaises(AttributeError, api.call_zone_method, + context, method) + + def test_call_zone_method_generates_exception(self): + context = {} + method = 'raises_exception' + results = api.call_zone_method(context, method) + + # FIXME(sirp): for now the _error_trap code is catching errors and + # converting them to a ("ERROR", "string") tuples. The code (and this + # test) should eventually handle real exceptions. + expected = [(1, ('ERROR', 'testing'))] + self.assertEqual(expected, results) From 5fa8101e567f1e1d181056354c146a065c7804b3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 12 May 2011 18:44:22 -0700 Subject: [PATCH 11/24] pep8 --- nova/tests/test_zone_aware_scheduler.py | 119 ++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 nova/tests/test_zone_aware_scheduler.py diff --git a/nova/tests/test_zone_aware_scheduler.py b/nova/tests/test_zone_aware_scheduler.py new file mode 100644 index 00000000..fdcde34c --- /dev/null +++ b/nova/tests/test_zone_aware_scheduler.py @@ -0,0 +1,119 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Zone Aware Scheduler. +""" + +from nova import test +from nova.scheduler import driver +from nova.scheduler import zone_aware_scheduler +from nova.scheduler import zone_manager + + +class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler): + def filter_hosts(self, num, specs): + # NOTE(sirp): this is returning [(hostname, services)] + return self.zone_manager.service_states.items() + + def weigh_hosts(self, num, specs, hosts): + fake_weight = 99 + weighted = [] + for hostname, caps in hosts: + weighted.append(dict(weight=fake_weight, name=hostname)) + return weighted + + +class FakeZoneManager(zone_manager.ZoneManager): + def __init__(self): + self.service_states = { + 'host1': { + 'compute': {'ram': 1000} + }, + 'host2': { + 'compute': {'ram': 2000} + }, + 'host3': { + 'compute': {'ram': 3000} + } + } + + +class FakeEmptyZoneManager(zone_manager.ZoneManager): + def __init__(self): + self.service_states = {} + + +def fake_empty_call_zone_method(context, method, specs): + return [] + + +def fake_call_zone_method(context, method, specs): + return [ + ('zone1', [ + dict(weight=1, blob='AAAAAAA'), + dict(weight=111, blob='BBBBBBB'), + dict(weight=112, blob='CCCCCCC'), + dict(weight=113, blob='DDDDDDD'), + ]), + ('zone2', [ + dict(weight=120, blob='EEEEEEE'), + dict(weight=2, blob='FFFFFFF'), + dict(weight=122, blob='GGGGGGG'), + dict(weight=123, blob='HHHHHHH'), + ]), + ('zone3', [ + dict(weight=130, blob='IIIIIII'), + dict(weight=131, blob='JJJJJJJ'), + dict(weight=132, blob='KKKKKKK'), + dict(weight=3, blob='LLLLLLL'), + ]), + ] + + +class ZoneAwareSchedulerTestCase(test.TestCase): + """Test case for Zone Aware Scheduler.""" + + def test_zone_aware_scheduler(self): + """ + Create a nested set of FakeZones, ensure that a select call returns the + appropriate build plan. + """ + sched = FakeZoneAwareScheduler() + self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method) + + zm = FakeZoneManager() + sched.set_zone_manager(zm) + + fake_context = {} + build_plan = sched.select(fake_context, {}) + + self.assertEqual(15, len(build_plan)) + + hostnames = [plan_item['name'] + for plan_item in build_plan if 'name' in plan_item] + self.assertEqual(3, len(hostnames)) + + def test_empty_zone_aware_scheduler(self): + """ + Ensure empty hosts & child_zones result in NoValidHosts exception. + """ + sched = FakeZoneAwareScheduler() + self.stubs.Set(sched, '_call_zone_method', fake_empty_call_zone_method) + + zm = FakeEmptyZoneManager() + sched.set_zone_manager(zm) + + fake_context = {} + self.assertRaises(driver.NoValidHost, sched.schedule, fake_context, {}) From fffb7787306e31dbe0fa4666f2d7f931d79d77f0 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 13 May 2011 06:12:18 -0700 Subject: [PATCH 12/24] fixup based on Lorin's feedback --- nova/scheduler/zone_aware_scheduler.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index 07f86450..b3d230bd 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -36,7 +36,7 @@ class ZoneAwareScheduler(driver.Scheduler): """Call novaclient zone method. Broken out for testing.""" return api.call_zone_method(context, method, specs=specs) - def schedule_run_instance(self, context, topic='compute', specs=None, + def schedule_run_instance(self, context, topic='compute', specs={}, *args, **kwargs): """This method is called from nova.compute.api to provision an instance. However we need to look at the parameters being @@ -54,6 +54,10 @@ class ZoneAwareScheduler(driver.Scheduler): for item in build_plan: self.provision_instance(context, topic, item) + def provision_instance(context, topic, item): + """Create the requested instance in this Zone or a child zone.""" + pass + def select(self, context, *args, **kwargs): """Select returns a list of weights and zone/host information corresponding to the best hosts to service the request. Any @@ -111,5 +115,5 @@ class ZoneAwareScheduler(driver.Scheduler): def weigh_hosts(self, num, specs, hosts): """Derived classes must override this method and return - a lists of hosts in [(weight, hostname)] format.""" + a lists of hosts in [{weight, hostname}] format.""" raise NotImplemented() From e6d2222f513b52618ad2bd2134f842d0f309988e Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Mon, 16 May 2011 18:14:09 +0400 Subject: [PATCH 13/24] Added response about error in nova-manage project operations --- bin/nova-manage | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index a36ec86d..155ab592 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -362,27 +362,47 @@ class ProjectCommands(object): def add(self, project_id, user_id): """Adds user to project arguments: project_id user_id""" - self.manager.add_to_project(user_id, project_id) + try: + self.manager.add_to_project(user_id, project_id) + except exception.UserNotFound, e: + print e + raise def create(self, name, project_manager, description=None): """Creates a new project arguments: name project_manager [description]""" - self.manager.create_project(name, project_manager, description) + try: + self.manager.create_project(name, project_manager, description) + except exception.UserNotFound, e: + print e + raise def modify(self, name, project_manager, description=None): """Modifies a project arguments: name project_manager [description]""" - self.manager.modify_project(name, project_manager, description) - + try: + self.manager.modify_project(name, project_manager, description) + except exception.UserNotFound, e: + print e + raise + def delete(self, name): """Deletes an existing project arguments: name""" - self.manager.delete_project(name) + try: + self.manager.delete_project(name) + except exception.ProjectNotFound, e: + print e + raise def environment(self, project_id, user_id, filename='novarc'): """Exports environment variables to an sourcable file arguments: project_id user_id [filename='novarc]""" - rc = self.manager.get_environment_rc(user_id, project_id) + try: + rc = self.manager.get_environment_rc(user_id, project_id) + except (exception.UserNotFound, exception.ProjectNotFound), e: + print e + raise with open(filename, 'w') as f: f.write(rc) @@ -400,7 +420,7 @@ class ProjectCommands(object): quo = {'project_id': project_id, key: value} try: db.quota_update(ctxt, project_id, quo) - except exception.NotFound: + except exception.ProjectQuotaNotFound: db.quota_create(ctxt, quo) project_quota = quota.get_quota(ctxt, project_id) for key, value in project_quota.iteritems(): @@ -409,7 +429,11 @@ class ProjectCommands(object): def remove(self, project_id, user_id): """Removes user from project arguments: project_id user_id""" - self.manager.remove_from_project(user_id, project_id) + try: + self.manager.remove_from_project(user_id, project_id) + except (exception.UserNotFound, exception.ProjectNotFound), e: + print e + raise def scrub(self, project_id): """Deletes data associated with project @@ -428,6 +452,9 @@ class ProjectCommands(object): zip_file = self.manager.get_credentials(user_id, project_id) with open(filename, 'w') as f: f.write(zip_file) + except (exception.UserNotFound, exception.ProjectNotFound), e: + print e + raise except db.api.NoMoreNetworks: print _('No more networks available. If this is a new ' 'installation, you need\nto call something like this:\n\n' From 04abb75adfce476c962e0b55417e1c488dda819a Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Mon, 16 May 2011 18:17:15 +0400 Subject: [PATCH 14/24] Pep8 cleaning --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 155ab592..f1214ff3 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -385,7 +385,7 @@ class ProjectCommands(object): except exception.UserNotFound, e: print e raise - + def delete(self, name): """Deletes an existing project arguments: name""" From 861f42f7bf7a1ec92cc5b59de754553d2587505d Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Mon, 16 May 2011 22:40:16 +0400 Subject: [PATCH 15/24] style fixing --- bin/nova-manage | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index f1214ff3..a42dabf8 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -364,8 +364,8 @@ class ProjectCommands(object): arguments: project_id user_id""" try: self.manager.add_to_project(user_id, project_id) - except exception.UserNotFound, e: - print e + except exception.UserNotFound as ex: + print ex raise def create(self, name, project_manager, description=None): @@ -373,8 +373,8 @@ class ProjectCommands(object): arguments: name project_manager [description]""" try: self.manager.create_project(name, project_manager, description) - except exception.UserNotFound, e: - print e + except exception.UserNotFound as ex: + print ex raise def modify(self, name, project_manager, description=None): @@ -382,8 +382,8 @@ class ProjectCommands(object): arguments: name project_manager [description]""" try: self.manager.modify_project(name, project_manager, description) - except exception.UserNotFound, e: - print e + except exception.UserNotFound as ex: + print ex raise def delete(self, name): @@ -391,8 +391,8 @@ class ProjectCommands(object): arguments: name""" try: self.manager.delete_project(name) - except exception.ProjectNotFound, e: - print e + except exception.ProjectNotFound as ex: + print ex raise def environment(self, project_id, user_id, filename='novarc'): @@ -400,8 +400,8 @@ class ProjectCommands(object): arguments: project_id user_id [filename='novarc]""" try: rc = self.manager.get_environment_rc(user_id, project_id) - except (exception.UserNotFound, exception.ProjectNotFound), e: - print e + except (exception.UserNotFound, exception.ProjectNotFound) as ex: + print ex raise with open(filename, 'w') as f: f.write(rc) @@ -431,8 +431,8 @@ class ProjectCommands(object): arguments: project_id user_id""" try: self.manager.remove_from_project(user_id, project_id) - except (exception.UserNotFound, exception.ProjectNotFound), e: - print e + except (exception.UserNotFound, exception.ProjectNotFound) as ex: + print ex raise def scrub(self, project_id): @@ -452,8 +452,8 @@ class ProjectCommands(object): zip_file = self.manager.get_credentials(user_id, project_id) with open(filename, 'w') as f: f.write(zip_file) - except (exception.UserNotFound, exception.ProjectNotFound), e: - print e + except (exception.UserNotFound, exception.ProjectNotFound) as ex: + print ex raise except db.api.NoMoreNetworks: print _('No more networks available. If this is a new ' From 27b71e79e12f3dbb27e40fccf0128f0f214ac324 Mon Sep 17 00:00:00 2001 From: Andrey Brindeyev Date: Fri, 20 May 2011 17:57:04 +0400 Subject: [PATCH 16/24] Addressing bug #785763. Usual default for maximum number of DHCP leases in dnsmasq is 150. This prevents instances to obtain IP addresses from DHCP in case we have more than 150 in our network. Adding myself to Authors. --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 546c9091..6741c81f 100644 --- a/Authors +++ b/Authors @@ -1,4 +1,5 @@ Alex Meade +Andrey Brindeyev Andy Smith Andy Southgate Anne Gentle From 7a3ecc4015d1d2198d0fa0662aecc06ba7cc5e9c Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 20 May 2011 21:21:04 +0200 Subject: [PATCH 17/24] Include data files for public key tests in the tarball. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index e7a6e7da..4e145de7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -35,6 +35,7 @@ include nova/tests/bundle/1mb.manifest.xml include nova/tests/bundle/1mb.no_kernel_or_ramdisk.manifest.xml include nova/tests/bundle/1mb.part.0 include nova/tests/bundle/1mb.part.1 +include nova/tests/public_key/* include nova/tests/db/nova.austin.sqlite include plugins/xenapi/README include plugins/xenapi/etc/xapi.d/plugins/objectstore From 713978756ce5ba29f444eec6478ffa177c85f232 Mon Sep 17 00:00:00 2001 From: "Dave Walker (Daviey)" Date: Sat, 21 May 2011 13:00:22 +0100 Subject: [PATCH 20/24] Added myself to Authors --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 6741c81f..e7bddd21 100644 --- a/Authors +++ b/Authors @@ -17,6 +17,7 @@ Christian Berendt Chuck Short Cory Wright Dan Prince +Dave Walker David Pravec Dean Troyer Devin Carlen From 1eb1bde49694ff143e9948dcc558a472285123a1 Mon Sep 17 00:00:00 2001 From: "Dave Walker (Daviey)" Date: Mon, 23 May 2011 22:15:10 +0100 Subject: [PATCH 21/24] Added test case for attempting to create a duplicate keypair --- nova/tests/test_api.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index 97f401b8..7c0331ef 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -224,6 +224,29 @@ class ApiEc2TestCase(test.TestCase): self.manager.delete_project(project) self.manager.delete_user(user) + def test_create_duplicate_key_pair(self): + """Test that, after successfully generating a keypair, + requesting a second keypair with the same name fails sanely""" + self.expect_http() + self.mox.ReplayAll() + keyname = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ + for x in range(random.randint(4, 8))) + user = self.manager.create_user('fake', 'fake', 'fake') + project = self.manager.create_project('fake', 'fake', 'fake') + # NOTE(vish): create depends on pool, so call helper directly + self.ec2.create_key_pair('test') + + try: + self.ec2.create_key_pair('test') + except EC2ResponseError, e: + if e.code == 'KeyPairExists': + pass + else: + self.fail("Unexpected EC2ResponseError: %s " + "(expected KeyPairExists)" % e.code) + else: + self.fail('Exception not raised.') + def test_get_all_security_groups(self): """Test that we can retrieve security groups""" self.expect_http() From db61c0c5d60694fb9dac5478dc741b53e380bebc Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 May 2011 13:19:09 -0700 Subject: [PATCH 22/24] Properly reparse flags when adding dynamic flags --- nova/flags.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 32cb6efa..7304700f 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -110,7 +110,7 @@ class FlagValues(gflags.FlagValues): return name in self.__dict__['__dirty'] def ClearDirty(self): - self.__dict__['__is_dirty'] = [] + self.__dict__['__dirty'] = [] def WasAlreadyParsed(self): return self.__dict__['__was_already_parsed'] @@ -119,11 +119,11 @@ class FlagValues(gflags.FlagValues): if '__stored_argv' not in self.__dict__: return new_flags = FlagValues(self) - for k in self.__dict__['__dirty']: + for k in self.FlagDict().iterkeys(): new_flags[k] = gflags.FlagValues.__getitem__(self, k) new_flags(self.__dict__['__stored_argv']) - for k in self.__dict__['__dirty']: + for k in new_flags.FlagDict().iterkeys(): setattr(self, k, getattr(new_flags, k)) self.ClearDirty() From d608b6e6bd31d9496da6e4ba4a42687fa33295fa Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 May 2011 13:19:09 -0700 Subject: [PATCH 23/24] add a test from vish and fix the issues --- nova/flags.py | 1 + nova/tests/test_flags.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/nova/flags.py b/nova/flags.py index 7304700f..9eaac559 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -122,6 +122,7 @@ class FlagValues(gflags.FlagValues): for k in self.FlagDict().iterkeys(): new_flags[k] = gflags.FlagValues.__getitem__(self, k) + new_flags.Reset() new_flags(self.__dict__['__stored_argv']) for k in new_flags.FlagDict().iterkeys(): setattr(self, k, getattr(new_flags, k)) diff --git a/nova/tests/test_flags.py b/nova/tests/test_flags.py index 707300fc..05319d91 100644 --- a/nova/tests/test_flags.py +++ b/nova/tests/test_flags.py @@ -91,6 +91,20 @@ class FlagsTestCase(test.TestCase): self.assert_('runtime_answer' in self.global_FLAGS) self.assertEqual(self.global_FLAGS.runtime_answer, 60) + def test_long_vs_short_flags(self): + flags.DEFINE_string('duplicate_answer_long', 'val', 'desc', + flag_values=self.global_FLAGS) + argv = ['flags_test', '--duplicate_answer=60', 'extra_arg'] + args = self.global_FLAGS(argv) + + self.assert_('duplicate_answer' not in self.global_FLAGS) + self.assert_(self.global_FLAGS.duplicate_answer_long, 60) + + flags.DEFINE_integer('duplicate_answer', 60, 'desc', + flag_values=self.global_FLAGS) + self.assertEqual(self.global_FLAGS.duplicate_answer, 60) + self.assertEqual(self.global_FLAGS.duplicate_answer_long, 'val') + def test_flag_leak_left(self): self.assertEqual(FLAGS.flags_unittest, 'foo') FLAGS.flags_unittest = 'bar' From 2878a53bb8bb7dee6e30a8a374f96f8a4572183d Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 24 May 2011 13:19:09 -0700 Subject: [PATCH 24/24] make fake_flags set defaults instead of runtime values --- bin/nova-dhcpbridge | 7 +++++++ nova/tests/fake_flags.py | 28 ++++++++++++++-------------- nova/tests/real_flags.py | 26 -------------------------- 3 files changed, 21 insertions(+), 40 deletions(-) delete mode 100644 nova/tests/real_flags.py diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index f42dfd6b..5926b97d 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -108,6 +108,13 @@ def main(): interface = os.environ.get('DNSMASQ_INTERFACE', FLAGS.dnsmasq_interface) if int(os.environ.get('TESTING', '0')): from nova.tests import fake_flags + + #if FLAGS.fake_rabbit: + # LOG.debug(_("leasing ip")) + # network_manager = utils.import_object(FLAGS.network_manager) + ## reload(fake_flags) + # from nova.tests import fake_flags + action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 5d7ca98b..ecefc464 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -21,24 +21,24 @@ from nova import flags FLAGS = flags.FLAGS flags.DECLARE('volume_driver', 'nova.volume.manager') -FLAGS.volume_driver = 'nova.volume.driver.FakeISCSIDriver' -FLAGS.connection_type = 'fake' -FLAGS.fake_rabbit = True +FLAGS['volume_driver'].SetDefault('nova.volume.driver.FakeISCSIDriver') +FLAGS['connection_type'].SetDefault('fake') +FLAGS['fake_rabbit'].SetDefault(True) flags.DECLARE('auth_driver', 'nova.auth.manager') -FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' +FLAGS['auth_driver'].SetDefault('nova.auth.dbdriver.DbDriver') flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') flags.DECLARE('fake_network', 'nova.network.manager') -FLAGS.network_size = 8 -FLAGS.num_networks = 2 -FLAGS.fake_network = True -FLAGS.image_service = 'nova.image.local.LocalImageService' +FLAGS['network_size'].SetDefault(8) +FLAGS['num_networks'].SetDefault(2) +FLAGS['fake_network'].SetDefault(True) +FLAGS['image_service'].SetDefault('nova.image.local.LocalImageService') flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') flags.DECLARE('iscsi_num_targets', 'nova.volume.driver') -FLAGS.num_shelves = 2 -FLAGS.blades_per_shelf = 4 -FLAGS.iscsi_num_targets = 8 -FLAGS.verbose = True -FLAGS.sqlite_db = "tests.sqlite" -FLAGS.use_ipv6 = True +FLAGS['num_shelves'].SetDefault(2) +FLAGS['blades_per_shelf'].SetDefault(4) +FLAGS['iscsi_num_targets'].SetDefault(8) +FLAGS['verbose'].SetDefault(True) +FLAGS['sqlite_db'].SetDefault("tests.sqlite") +FLAGS['use_ipv6'].SetDefault(True) diff --git a/nova/tests/real_flags.py b/nova/tests/real_flags.py deleted file mode 100644 index 71da0499..00000000 --- a/nova/tests/real_flags.py +++ /dev/null @@ -1,26 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from nova import flags - -FLAGS = flags.FLAGS - -FLAGS.connection_type = 'libvirt' -FLAGS.fake_rabbit = False -FLAGS.fake_network = False -FLAGS.verbose = False