From e283358ca051dbb02e13b47a8c8ef3ed8d769d5d Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Wed, 25 May 2011 15:48:03 -0400 Subject: [PATCH 01/40] Initial tests --- nova/tests/test_instance_types_metadata.py | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 nova/tests/test_instance_types_metadata.py diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py new file mode 100644 index 00000000..a22bf2b7 --- /dev/null +++ b/nova/tests/test_instance_types_metadata.py @@ -0,0 +1,62 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 University of Southern California +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Unit Tests for instance types metadata code +""" + + +from nova import test +from nova.db.sqlalchemy.session import get_session +from nova.db.sqlalchemy import models + + +class InstanceTypeMetadataTestCase(test.TestCase): + + def setUp(self): + super(InstanceTypeMetadataTestCase, self).setUp() + values = dict(memory_mb=22000, + vcpus=8, + local_gb=1690, + flavorid=105) + metadata = dict(cpu_arch="x86_64", + cpu_info=dict(model="Nehalem"), + xpu_arch="fermi", + xpus=2, + xpu_info=dict(model="Tesla 2050", gcores="448"), + net_arch="ethernet", + net_mbps=10000) + + metadata_refs = [] + for k,v in metadata.iteritems(): + metadata_ref = models.InstanceTypeMetadata() + metadata_ref['key'] = k + metadata_ref['value'] = v + metadata_refs.append(metadata_ref) + values['metadata'] = metadata_refs + + instance_type_ref = models.InstanceTypes() + instance_type_ref.update(values) + + session = get_session() + with session.begin(): + instance_type_ref.save(session=session) + # Add cg1.4xlarge + + + def test_foo(self): + # Add a new instance type cg1 + # Add the metadata + # Retrieve the metadata + pass From 076b9e3b33c5a66e332cd2cc6b36194772029ed0 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:10:19 -0400 Subject: [PATCH 02/40] Adding test code --- nova/tests/test_instance_types_metadata.py | 26 +++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index a22bf2b7..06c11a7c 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -16,7 +16,9 @@ Unit Tests for instance types metadata code """ +import nova.db.api +from nova import context from nova import test from nova.db.sqlalchemy.session import get_session from nova.db.sqlalchemy import models @@ -26,15 +28,16 @@ class InstanceTypeMetadataTestCase(test.TestCase): def setUp(self): super(InstanceTypeMetadataTestCase, self).setUp() - values = dict(memory_mb=22000, + values = dict(name="cg1.4xlarge", + memory_mb=22000, vcpus=8, local_gb=1690, flavorid=105) metadata = dict(cpu_arch="x86_64", - cpu_info=dict(model="Nehalem"), + cpu_model="Nehalem", xpu_arch="fermi", xpus=2, - xpu_info=dict(model="Tesla 2050", gcores="448"), + xpu_model="Tesla 2050", net_arch="ethernet", net_mbps=10000) @@ -44,19 +47,22 @@ class InstanceTypeMetadataTestCase(test.TestCase): metadata_ref['key'] = k metadata_ref['value'] = v metadata_refs.append(metadata_ref) - values['metadata'] = metadata_refs + values['meta'] = metadata_refs instance_type_ref = models.InstanceTypes() instance_type_ref.update(values) + session = get_session() with session.begin(): instance_type_ref.save(session=session) - # Add cg1.4xlarge + self.instance_type_id = instance_type_ref.id + def test_instance_type_metadata_get(self): + self.assertEquals( \ + nova.db.api.instance_type_metadata_get(context.get_admin_context(), + self.instance_type_id), + {'foo' : 'bar'}) - def test_foo(self): - # Add a new instance type cg1 - # Add the metadata - # Retrieve the metadata - pass + + \ No newline at end of file From 518b779e8bc74870099bf296e130956a67979a34 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:29:51 -0400 Subject: [PATCH 03/40] Added a unit test --- nova/tests/test_instance_types_metadata.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index 06c11a7c..5d8364c3 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -16,9 +16,8 @@ Unit Tests for instance types metadata code """ -import nova.db.api - from nova import context +from nova import db from nova import test from nova.db.sqlalchemy.session import get_session from nova.db.sqlalchemy import models @@ -59,10 +58,17 @@ class InstanceTypeMetadataTestCase(test.TestCase): self.instance_type_id = instance_type_ref.id def test_instance_type_metadata_get(self): - self.assertEquals( \ - nova.db.api.instance_type_metadata_get(context.get_admin_context(), - self.instance_type_id), - {'foo' : 'bar'}) + expected_metadata = dict(cpu_arch="x86_64", + cpu_model="Nehalem", + xpu_arch="fermi", + xpus="2", + xpu_model="Tesla 2050", + net_arch="ethernet", + net_mbps="10000") + retrieved_metadata = db.api.instance_type_metadata_get( + context.get_admin_context(), + self.instance_type_id) + self.assertEquals(expected_metadata, retrieved_metadata) \ No newline at end of file From 79c73fbc65334646c8c60d907cbe93fc4b1d8cda Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:39:37 -0400 Subject: [PATCH 04/40] Added delete instance metadata unit test --- nova/tests/test_instance_types_metadata.py | 25 +++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index 5d8364c3..c83c5bfc 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -47,11 +47,8 @@ class InstanceTypeMetadataTestCase(test.TestCase): metadata_ref['value'] = v metadata_refs.append(metadata_ref) values['meta'] = metadata_refs - instance_type_ref = models.InstanceTypes() instance_type_ref.update(values) - - session = get_session() with session.begin(): instance_type_ref.save(session=session) @@ -65,10 +62,22 @@ class InstanceTypeMetadataTestCase(test.TestCase): xpu_model="Tesla 2050", net_arch="ethernet", net_mbps="10000") - retrieved_metadata = db.api.instance_type_metadata_get( + actual_metadata = db.api.instance_type_metadata_get( context.get_admin_context(), self.instance_type_id) - self.assertEquals(expected_metadata, retrieved_metadata) - - - \ No newline at end of file + self.assertEquals(expected_metadata, actual_metadata) + + def test_instance_type_metadata_delete(self): + expected_metadata = dict(cpu_arch="x86_64", + cpu_model="Nehalem", + xpu_arch="fermi", + xpus="2", + net_arch="ethernet", + net_mbps="10000") + db.api.instance_type_metadata_delete(context.get_admin_context(), + self.instance_type_id, + "xpu_model") + actual_metadata = db.api.instance_type_metadata_get( + context.get_admin_context(), + self.instance_type_id) + self.assertEquals(expected_metadata, actual_metadata) \ No newline at end of file From c02bbf3cc4b5aa83a6fa54a64bf1ddd752394fef Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:44:40 -0400 Subject: [PATCH 05/40] Added test for instance type metadata update --- nova/tests/test_instance_types_metadata.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index c83c5bfc..d72a72e0 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -80,4 +80,24 @@ class InstanceTypeMetadataTestCase(test.TestCase): actual_metadata = db.api.instance_type_metadata_get( context.get_admin_context(), self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) \ No newline at end of file + self.assertEquals(expected_metadata, actual_metadata) + + def test_instance_type_metadata_update(self): + expected_metadata = dict(cpu_arch="x86_64", + cpu_model="Sandy Bridge", + xpu_arch="fermi", + xpus="2", + xpu_model="Tesla 2050", + net_arch="ethernet", + net_mbps="10000") + db.api.instance_type_metadata_update_or_create( + context.get_admin_context(), + self.instance_type_id, + dict(cpu_model="Sandy Bridge")) + actual_metadata = db.api.instance_type_metadata_get( + context.get_admin_context(), + self.instance_type_id) + self.assertEquals(expected_metadata, actual_metadata) + + + \ No newline at end of file From 3c63c854129ff069645ba356a3a6fb8ed13e931f Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 13:54:19 -0400 Subject: [PATCH 06/40] Added test for instance type metadata create --- nova/tests/test_instance_types_metadata.py | 44 ++++++++++++++-------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index d72a72e0..085a951a 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -36,10 +36,7 @@ class InstanceTypeMetadataTestCase(test.TestCase): cpu_model="Nehalem", xpu_arch="fermi", xpus=2, - xpu_model="Tesla 2050", - net_arch="ethernet", - net_mbps=10000) - + xpu_model="Tesla 2050") metadata_refs = [] for k,v in metadata.iteritems(): metadata_ref = models.InstanceTypeMetadata() @@ -54,14 +51,18 @@ class InstanceTypeMetadataTestCase(test.TestCase): instance_type_ref.save(session=session) self.instance_type_id = instance_type_ref.id + def tearDown(self): + # Remove the instance from the database + db.api.instance_type_purge(context.get_admin_context(), "cg1.4xlarge") + super(InstanceTypeMetadataTestCase, self).tearDown() + + def test_instance_type_metadata_get(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", xpu_arch="fermi", xpus="2", - xpu_model="Tesla 2050", - net_arch="ethernet", - net_mbps="10000") + xpu_model="Tesla 2050") actual_metadata = db.api.instance_type_metadata_get( context.get_admin_context(), self.instance_type_id) @@ -71,9 +72,7 @@ class InstanceTypeMetadataTestCase(test.TestCase): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", xpu_arch="fermi", - xpus="2", - net_arch="ethernet", - net_mbps="10000") + xpus="2") db.api.instance_type_metadata_delete(context.get_admin_context(), self.instance_type_id, "xpu_model") @@ -87,9 +86,7 @@ class InstanceTypeMetadataTestCase(test.TestCase): cpu_model="Sandy Bridge", xpu_arch="fermi", xpus="2", - xpu_model="Tesla 2050", - net_arch="ethernet", - net_mbps="10000") + xpu_model="Tesla 2050") db.api.instance_type_metadata_update_or_create( context.get_admin_context(), self.instance_type_id, @@ -98,6 +95,21 @@ class InstanceTypeMetadataTestCase(test.TestCase): context.get_admin_context(), self.instance_type_id) self.assertEquals(expected_metadata, actual_metadata) - - - \ No newline at end of file + + def test_instance_type_metadata_create(self): + expected_metadata = dict(cpu_arch="x86_64", + cpu_model="Nehalem", + xpu_arch="fermi", + xpus="2", + xpu_model="Tesla 2050", + net_arch="ethernet", + net_mbps="10000") + db.api.instance_type_metadata_update_or_create( + context.get_admin_context(), + self.instance_type_id, + dict(net_arch="ethernet", + net_mbps=10000)) + actual_metadata = db.api.instance_type_metadata_get( + context.get_admin_context(), + self.instance_type_id) + self.assertEquals(expected_metadata, actual_metadata) From 9a3e0294bfd58ca7d3f49d6d51afbef7dff780eb Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 14:20:08 -0400 Subject: [PATCH 07/40] Modified instance_type_create to take metadata --- nova/tests/test_instance_types_metadata.py | 23 +++++++--------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index 085a951a..5263b1cb 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -27,6 +27,7 @@ class InstanceTypeMetadataTestCase(test.TestCase): def setUp(self): super(InstanceTypeMetadataTestCase, self).setUp() + self.context = context.get_admin_context() values = dict(name="cg1.4xlarge", memory_mb=22000, vcpus=8, @@ -36,27 +37,17 @@ class InstanceTypeMetadataTestCase(test.TestCase): cpu_model="Nehalem", xpu_arch="fermi", xpus=2, - xpu_model="Tesla 2050") - metadata_refs = [] - for k,v in metadata.iteritems(): - metadata_ref = models.InstanceTypeMetadata() - metadata_ref['key'] = k - metadata_ref['value'] = v - metadata_refs.append(metadata_ref) - values['meta'] = metadata_refs - instance_type_ref = models.InstanceTypes() - instance_type_ref.update(values) - session = get_session() - with session.begin(): - instance_type_ref.save(session=session) - self.instance_type_id = instance_type_ref.id + xpu_model="Tesla 2050") + values['meta'] = metadata + ref = db.api.instance_type_create(self.context, + values) + self.instance_type_id = ref.id def tearDown(self): - # Remove the instance from the database + # Remove the instance type from the database db.api.instance_type_purge(context.get_admin_context(), "cg1.4xlarge") super(InstanceTypeMetadataTestCase, self).tearDown() - def test_instance_type_metadata_get(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", From 5a2a97874f031eb2e08d8ba1be9b5175a164743c Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 27 May 2011 14:29:23 -0400 Subject: [PATCH 08/40] Fixing pep8 problems --- nova/tests/test_instance_types_metadata.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py index 5263b1cb..1c418588 100644 --- a/nova/tests/test_instance_types_metadata.py +++ b/nova/tests/test_instance_types_metadata.py @@ -24,7 +24,7 @@ from nova.db.sqlalchemy import models class InstanceTypeMetadataTestCase(test.TestCase): - + def setUp(self): super(InstanceTypeMetadataTestCase, self).setUp() self.context = context.get_admin_context() @@ -42,12 +42,12 @@ class InstanceTypeMetadataTestCase(test.TestCase): ref = db.api.instance_type_create(self.context, values) self.instance_type_id = ref.id - + def tearDown(self): # Remove the instance type from the database db.api.instance_type_purge(context.get_admin_context(), "cg1.4xlarge") super(InstanceTypeMetadataTestCase, self).tearDown() - + def test_instance_type_metadata_get(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", @@ -58,20 +58,20 @@ class InstanceTypeMetadataTestCase(test.TestCase): context.get_admin_context(), self.instance_type_id) self.assertEquals(expected_metadata, actual_metadata) - + def test_instance_type_metadata_delete(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", xpu_arch="fermi", xpus="2") - db.api.instance_type_metadata_delete(context.get_admin_context(), + db.api.instance_type_metadata_delete(context.get_admin_context(), self.instance_type_id, "xpu_model") actual_metadata = db.api.instance_type_metadata_get( context.get_admin_context(), self.instance_type_id) self.assertEquals(expected_metadata, actual_metadata) - + def test_instance_type_metadata_update(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Sandy Bridge", @@ -86,13 +86,13 @@ class InstanceTypeMetadataTestCase(test.TestCase): context.get_admin_context(), self.instance_type_id) self.assertEquals(expected_metadata, actual_metadata) - + def test_instance_type_metadata_create(self): expected_metadata = dict(cpu_arch="x86_64", cpu_model="Nehalem", xpu_arch="fermi", xpus="2", - xpu_model="Tesla 2050", + xpu_model="Tesla 2050", net_arch="ethernet", net_mbps="10000") db.api.instance_type_metadata_update_or_create( From 02ced950567c366b04f617ba73d3c05e50795cb3 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Thu, 16 Jun 2011 16:33:29 -0400 Subject: [PATCH 10/40] Rename: intance_type_metadata -> instance_type_extra_specs --- nova/tests/test_instance_types_metadata.py | 106 --------------------- 1 file changed, 106 deletions(-) delete mode 100644 nova/tests/test_instance_types_metadata.py diff --git a/nova/tests/test_instance_types_metadata.py b/nova/tests/test_instance_types_metadata.py deleted file mode 100644 index 1c418588..00000000 --- a/nova/tests/test_instance_types_metadata.py +++ /dev/null @@ -1,106 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 University of Southern California -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Unit Tests for instance types metadata code -""" - -from nova import context -from nova import db -from nova import test -from nova.db.sqlalchemy.session import get_session -from nova.db.sqlalchemy import models - - -class InstanceTypeMetadataTestCase(test.TestCase): - - def setUp(self): - super(InstanceTypeMetadataTestCase, self).setUp() - self.context = context.get_admin_context() - values = dict(name="cg1.4xlarge", - memory_mb=22000, - vcpus=8, - local_gb=1690, - flavorid=105) - metadata = dict(cpu_arch="x86_64", - cpu_model="Nehalem", - xpu_arch="fermi", - xpus=2, - xpu_model="Tesla 2050") - values['meta'] = metadata - ref = db.api.instance_type_create(self.context, - values) - self.instance_type_id = ref.id - - def tearDown(self): - # Remove the instance type from the database - db.api.instance_type_purge(context.get_admin_context(), "cg1.4xlarge") - super(InstanceTypeMetadataTestCase, self).tearDown() - - def test_instance_type_metadata_get(self): - expected_metadata = dict(cpu_arch="x86_64", - cpu_model="Nehalem", - xpu_arch="fermi", - xpus="2", - xpu_model="Tesla 2050") - actual_metadata = db.api.instance_type_metadata_get( - context.get_admin_context(), - self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) - - def test_instance_type_metadata_delete(self): - expected_metadata = dict(cpu_arch="x86_64", - cpu_model="Nehalem", - xpu_arch="fermi", - xpus="2") - db.api.instance_type_metadata_delete(context.get_admin_context(), - self.instance_type_id, - "xpu_model") - actual_metadata = db.api.instance_type_metadata_get( - context.get_admin_context(), - self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) - - def test_instance_type_metadata_update(self): - expected_metadata = dict(cpu_arch="x86_64", - cpu_model="Sandy Bridge", - xpu_arch="fermi", - xpus="2", - xpu_model="Tesla 2050") - db.api.instance_type_metadata_update_or_create( - context.get_admin_context(), - self.instance_type_id, - dict(cpu_model="Sandy Bridge")) - actual_metadata = db.api.instance_type_metadata_get( - context.get_admin_context(), - self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) - - def test_instance_type_metadata_create(self): - expected_metadata = dict(cpu_arch="x86_64", - cpu_model="Nehalem", - xpu_arch="fermi", - xpus="2", - xpu_model="Tesla 2050", - net_arch="ethernet", - net_mbps="10000") - db.api.instance_type_metadata_update_or_create( - context.get_admin_context(), - self.instance_type_id, - dict(net_arch="ethernet", - net_mbps=10000)) - actual_metadata = db.api.instance_type_metadata_get( - context.get_admin_context(), - self.instance_type_id) - self.assertEquals(expected_metadata, actual_metadata) From b7fe156f3de0f8b0e3d395ec814317b12335db47 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sat, 18 Jun 2011 23:10:41 -0400 Subject: [PATCH 11/40] General cleanup and refactor of a lot of the API/WSGI service code. --- bin/nova-api | 18 ++++++++++++++++-- nova/log.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index a1088c23..6db68be9 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -24,6 +24,8 @@ import gettext import os import sys +import eventlet.pool + # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), @@ -46,6 +48,13 @@ LOG = logging.getLogger('nova.api') FLAGS = flags.FLAGS + +def launch(service_name): + _service = service.WSGIService(service_name) + _service.start() + _service.wait() + + if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) @@ -57,5 +66,10 @@ if __name__ == '__main__': flag_get = FLAGS.get(flag, None) LOG.debug("%(flag)s : %(flag_get)s" % locals()) - service = service.serve_wsgi(service.ApiService) - service.wait() + + pool = eventlet.pool.Pool() + pool.execute(launch, "ec2") + pool.execute(launch, "osapi") + pool.wait_all() + + print >>sys.stderr, "Exiting..." diff --git a/nova/log.py b/nova/log.py index 6909916a..d3bffdb2 100644 --- a/nova/log.py +++ b/nova/log.py @@ -314,3 +314,16 @@ logging.setLoggerClass(NovaLogger) def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" logging.root.log(AUDIT, msg, *args, **kwargs) + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.DEBUG): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg) + + From 056e48767cb10c1813f1411984ace8689b31bcc6 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 14:09:09 -0400 Subject: [PATCH 12/40] Removed debugging, made objectstore tests pass again. --- bin/nova-api | 3 --- nova/log.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 6db68be9..90c8b69a 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -66,10 +66,7 @@ if __name__ == '__main__': flag_get = FLAGS.get(flag, None) LOG.debug("%(flag)s : %(flag_get)s" % locals()) - pool = eventlet.pool.Pool() pool.execute(launch, "ec2") pool.execute(launch, "osapi") pool.wait_all() - - print >>sys.stderr, "Exiting..." diff --git a/nova/log.py b/nova/log.py index d3bffdb2..3080daee 100644 --- a/nova/log.py +++ b/nova/log.py @@ -325,5 +325,3 @@ class WritableLogger(object): def write(self, msg): self.logger.log(self.level, msg) - - From 6b1029efbe274db586b11dff81400f44d50222c3 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 14:41:42 -0400 Subject: [PATCH 13/40] Cleaned up nova-api binary and logging a bit. --- bin/nova-api | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 90c8b69a..7d80b0b7 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -20,33 +20,20 @@ """Starter script for Nova API.""" -import gettext -import os import sys import eventlet.pool -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - from nova import flags from nova import log as logging from nova import service from nova import utils from nova import version -from nova import wsgi -LOG = logging.getLogger('nova.api') - +LOG = logging.getLogger("nova.api") FLAGS = flags.FLAGS +VERSION = version.version_string_with_vcs() def launch(service_name): @@ -55,18 +42,17 @@ def launch(service_name): _service.wait() -if __name__ == '__main__': +def main(): utils.default_flagfile() FLAGS(sys.argv) - logging.setup() - LOG.audit(_("Starting nova-api node (version %s)"), - version.version_string_with_vcs()) - LOG.debug(_("Full set of FLAGS:")) - for flag in FLAGS: - flag_get = FLAGS.get(flag, None) - LOG.debug("%(flag)s : %(flag_get)s" % locals()) +# logging.setup() + LOG.audit(_("Starting nova-api node (version %s)") % VERSION) pool = eventlet.pool.Pool() pool.execute(launch, "ec2") pool.execute(launch, "osapi") pool.wait_all() + + +if __name__ == '__main__': + sys.exit(main()) From ffaa6f28a936e5f15a17bcb291d15059dc83945c Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 16:27:46 -0400 Subject: [PATCH 14/40] Further nova-api cleanup. --- bin/nova-api | 65 ++++++++++++++++++++++++++++++---------------------- nova/log.py | 2 +- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 7d80b0b7..ff4aa83d 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,7 +1,5 @@ #!/usr/bin/env python -# pylint: disable=C0103 -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - +# # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -18,40 +16,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Starter script for Nova API.""" +"""Starter script for Nova API. + +Starts both the EC2 and OpenStack APIs in separate processes. Pylint warnings +about re-imports should be ignored. + +""" + +# pylint: disable=W0404 import sys +import multiprocessing -import eventlet.pool - -from nova import flags -from nova import log as logging -from nova import service -from nova import utils -from nova import version - - -LOG = logging.getLogger("nova.api") -FLAGS = flags.FLAGS -VERSION = version.version_string_with_vcs() +import nova.flags +import nova.log +import nova.service +import nova.version +import nova.utils def launch(service_name): - _service = service.WSGIService(service_name) - _service.start() - _service.wait() + """Launch WSGI service with name matching 'paste' config file section.""" + service = nova.service.WSGIService(service_name) + service.start() + try: + service.wait() + except KeyboardInterrupt: + service.stop() def main(): - utils.default_flagfile() - FLAGS(sys.argv) -# logging.setup() - LOG.audit(_("Starting nova-api node (version %s)") % VERSION) + """Begin process of launching both EC2 and OSAPI services.""" + version = nova.version.version_string_with_vcs() + logger = nova.log.getLogger("nova.api") + logger.audit(_("Starting nova-api node (version %s)") % version) - pool = eventlet.pool.Pool() - pool.execute(launch, "ec2") - pool.execute(launch, "osapi") - pool.wait_all() + nova.flags.FLAGS(sys.argv) + nova.utils.default_flagfile() + + pool = multiprocessing.Pool(2) + pool.map_async(launch, ["ec2", "osapi"]) + pool.close() + + try: + pool.join() + except KeyboardInterrupt: + logger.audit(_("Exiting...")) + pool.terminate() if __name__ == '__main__': diff --git a/nova/log.py b/nova/log.py index 3080daee..f8c0ba68 100644 --- a/nova/log.py +++ b/nova/log.py @@ -319,7 +319,7 @@ def audit(msg, *args, **kwargs): class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" - def __init__(self, logger, level=logging.DEBUG): + def __init__(self, logger, level=logging.INFO): self.logger = logger self.level = level From 76d5b3f03de9c64decfd27e6b9febb22272791d2 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 18:00:38 -0400 Subject: [PATCH 15/40] Cleanup of the cleanup. --- bin/nova-api | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index ff4aa83d..1fcda24f 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,4 +1,5 @@ #!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -18,13 +19,10 @@ """Starter script for Nova API. -Starts both the EC2 and OpenStack APIs in separate processes. Pylint warnings -about re-imports should be ignored. +Starts both the EC2 and OpenStack APIs in separate processes. """ -# pylint: disable=W0404 - import sys import multiprocessing From db798a12a077735269e4bc36fd64a7a55c9bac07 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Sun, 19 Jun 2011 20:18:29 -0400 Subject: [PATCH 16/40] Introduced Loader concept, for paste decouple. --- bin/nova-api | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/nova-api b/bin/nova-api index 1fcda24f..885fb0ba 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -51,6 +51,8 @@ def main(): nova.flags.FLAGS(sys.argv) nova.utils.default_flagfile() + +# launch("osapi") pool = multiprocessing.Pool(2) pool.map_async(launch, ["ec2", "osapi"]) From 09bba14ef76c0131d574f9e3d5b9a3fa114386be Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 10:12:43 -0400 Subject: [PATCH 17/40] Cleanup and addition of tests for WSGI server. --- bin/nova-api | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index 885fb0ba..89112a15 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,6 +1,6 @@ #!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# + # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. From aee6affeffb79acc75fdefaa75ffb331e4d51e56 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 14:54:53 -0400 Subject: [PATCH 18/40] pep8 fixes --- bin/nova-api | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 89112a15..2345b3f2 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -51,8 +51,6 @@ def main(): nova.flags.FLAGS(sys.argv) nova.utils.default_flagfile() - -# launch("osapi") pool = multiprocessing.Pool(2) pool.map_async(launch, ["ec2", "osapi"]) From 5630fae25da028de6a7af87c4805f6dbc8e80acf Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 20 Jun 2011 19:32:18 -0400 Subject: [PATCH 19/40] Removed logging logic from __init__, added concept of Launcher...no tests for it yet. --- bin/nova-api | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 2345b3f2..563d7c09 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -24,44 +24,26 @@ Starts both the EC2 and OpenStack APIs in separate processes. """ import sys -import multiprocessing -import nova.flags import nova.log import nova.service import nova.version -import nova.utils - - -def launch(service_name): - """Launch WSGI service with name matching 'paste' config file section.""" - service = nova.service.WSGIService(service_name) - service.start() - try: - service.wait() - except KeyboardInterrupt: - service.stop() def main(): - """Begin process of launching both EC2 and OSAPI services.""" - version = nova.version.version_string_with_vcs() - logger = nova.log.getLogger("nova.api") - logger.audit(_("Starting nova-api node (version %s)") % version) - - nova.flags.FLAGS(sys.argv) - nova.utils.default_flagfile() - - pool = multiprocessing.Pool(2) - pool.map_async(launch, ["ec2", "osapi"]) - pool.close() + """Launch EC2 and OSAPI services.""" + ec2 = nova.service.WSGIService("ec2") + osapi = nova.service.WSGIService("osapi") + launcher = nova.service.Launcher(sys.argv) + launcher.launch_service(ec2) + launcher.launch_service(osapi) + try: - pool.join() + launcher.wait() except KeyboardInterrupt: - logger.audit(_("Exiting...")) - pool.terminate() - + launcher.stop() + if __name__ == '__main__': sys.exit(main()) From dc9cfe2f167befb2d56223850ca1f86b48230c9d Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:21:33 -0400 Subject: [PATCH 20/40] Tests for WSGI/Launcher --- bin/nova-api | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 563d7c09..20ad4bfa 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -38,12 +38,12 @@ def main(): launcher = nova.service.Launcher(sys.argv) launcher.launch_service(ec2) launcher.launch_service(osapi) - + try: launcher.wait() except KeyboardInterrupt: launcher.stop() - + if __name__ == '__main__': sys.exit(main()) From 99a91caee7b88a7d74ad5362092015b97070526f Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 00:32:31 -0400 Subject: [PATCH 21/40] Removed unneeded import. --- bin/nova-api | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index 20ad4bfa..b94928c7 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -27,7 +27,6 @@ import sys import nova.log import nova.service -import nova.version def main(): From 330f470992bd74079b4fb74fa87fdc7244f62365 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 11:25:44 -0400 Subject: [PATCH 22/40] Very small alterations, switched from using start() to pass host/port, to just defining them up front in init. Doesn't make sense to set them in start because we can't start more than once any way. Also, unbroke binaries. --- bin/nova-ajax-console-proxy | 5 +++-- bin/nova-direct-api | 7 +++++-- bin/nova-objectstore | 7 +++++-- bin/nova-vncproxy | 7 +++++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index d88f59e4..21cf6800 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -137,8 +137,9 @@ if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) logging.setup() - server = wsgi.Server() + acp_port = FLAGS.ajax_console_proxy_port acp = AjaxConsoleProxy() acp.register_listeners() - server.start(acp, FLAGS.ajax_console_proxy_port, host='0.0.0.0') + server = wsgi.Server("AJAX Console Proxy", acp, port=acp_port) + server.start() server.wait() diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 83ec7272..5d63eb87 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -93,6 +93,9 @@ if __name__ == '__main__': with_req = direct.PostParamsMiddleware(with_json) with_auth = direct.DelegatedAuthMiddleware(with_req) - server = wsgi.Server() - server.start(with_auth, FLAGS.direct_port, host=FLAGS.direct_host) + server = wsgi.Server("Direct API", + with_auth, + host=FLAGS.direct_host, + port=FLAGS.direct_port) + server.start() server.wait() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 6ef841b8..aa0dc063 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -50,6 +50,9 @@ if __name__ == '__main__': FLAGS(sys.argv) logging.setup() router = s3server.S3Application(FLAGS.buckets_path) - server = wsgi.Server() - server.start(router, FLAGS.s3_port, host=FLAGS.s3_host) + server = wsgi.Server("S3 Objectstore", + router, + port=FLAGS.s3_port, + host=FLAGS.s3_host) + server.start() server.wait() diff --git a/bin/nova-vncproxy b/bin/nova-vncproxy index ccb97e3a..62d3b948 100755 --- a/bin/nova-vncproxy +++ b/bin/nova-vncproxy @@ -96,6 +96,9 @@ if __name__ == "__main__": service.serve() - server = wsgi.Server() - server.start(with_auth, FLAGS.vncproxy_port, host=FLAGS.vncproxy_host) + server = wsgi.Server("VNC Proxy", + with_auth, + host=FLAGS.vncproxy_host, + port=FLAGS.vncproxy_port) + server.start() server.wait() From d5a5df9b839d164d72c6a089946b2736dc790c6b Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 21 Jun 2011 12:03:27 -0400 Subject: [PATCH 23/40] Removed whitespace. --- bin/nova-direct-api | 4 ++-- bin/nova-objectstore | 6 +++--- bin/nova-vncproxy | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 5d63eb87..c6cf9b2f 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -93,8 +93,8 @@ if __name__ == '__main__': with_req = direct.PostParamsMiddleware(with_json) with_auth = direct.DelegatedAuthMiddleware(with_req) - server = wsgi.Server("Direct API", - with_auth, + server = wsgi.Server("Direct API", + with_auth, host=FLAGS.direct_host, port=FLAGS.direct_port) server.start() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index aa0dc063..1aef3a25 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -50,9 +50,9 @@ if __name__ == '__main__': FLAGS(sys.argv) logging.setup() router = s3server.S3Application(FLAGS.buckets_path) - server = wsgi.Server("S3 Objectstore", - router, - port=FLAGS.s3_port, + server = wsgi.Server("S3 Objectstore", + router, + port=FLAGS.s3_port, host=FLAGS.s3_host) server.start() server.wait() diff --git a/bin/nova-vncproxy b/bin/nova-vncproxy index 62d3b948..72271df3 100755 --- a/bin/nova-vncproxy +++ b/bin/nova-vncproxy @@ -96,7 +96,7 @@ if __name__ == "__main__": service.serve() - server = wsgi.Server("VNC Proxy", + server = wsgi.Server("VNC Proxy", with_auth, host=FLAGS.vncproxy_host, port=FLAGS.vncproxy_port) From 1e2acef020d2c48f2d65a0ad72c7efa695d5ca58 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 22 Jun 2011 23:27:49 -0400 Subject: [PATCH 24/40] run launcher first since it initializes global flags and logging --- bin/nova-api | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index b94928c7..121f6f9a 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -31,10 +31,11 @@ import nova.service def main(): """Launch EC2 and OSAPI services.""" + launcher = nova.service.Launcher(sys.argv) + ec2 = nova.service.WSGIService("ec2") osapi = nova.service.WSGIService("osapi") - launcher = nova.service.Launcher(sys.argv) launcher.launch_service(ec2) launcher.launch_service(osapi) From d3648e45ea5b9576532f155b0bf09b9163d93f4c Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Thu, 23 Jun 2011 21:31:00 -0400 Subject: [PATCH 25/40] Created Bootstrapper to handle Nova bootstrapping logic. --- bin/nova-api | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 121f6f9a..ea99a1b4 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -25,17 +25,18 @@ Starts both the EC2 and OpenStack APIs in separate processes. import sys -import nova.log import nova.service +import nova.utils def main(): """Launch EC2 and OSAPI services.""" - launcher = nova.service.Launcher(sys.argv) + nova.utils.Bootstrapper.bootstrap_binary(sys.argv) ec2 = nova.service.WSGIService("ec2") osapi = nova.service.WSGIService("osapi") + launcher = nova.service.Launcher() launcher.launch_service(ec2) launcher.launch_service(osapi) From 101c2b0a9cdf8e9c5ffeae51747449655cefe025 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 24 Jun 2011 01:34:47 -0400 Subject: [PATCH 26/40] pep8 fixes --- nova/scheduler/host_filter.py | 24 ++++++++++++++++++++++-- nova/tests/scheduler/test_host_filter.py | 24 +++++++++++++++++++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index bd6b2660..7d4b9ad9 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -93,6 +93,22 @@ class InstanceTypeFilter(HostFilter): """Use instance_type to filter hosts.""" return (self._full_name(), instance_type) + def _satisfies_extra_specs(self, capabilities, instance_type): + """Check that the capabilities provided by the compute service + satisfy the extra specs associated with the instance type""" + + # Note(lorinh): For now, we are just checking exact matching on the + # values. Later on, we want to handle numerical + # values so we can represent things like number of GPU cards + try: + for key, value in instance_type['extra_specs'].iteritems(): + if capabilities[key] != value: + return False + except KeyError: + return False + + return True + def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" instance_type = query @@ -103,8 +119,12 @@ class InstanceTypeFilter(HostFilter): disk_bytes = capabilities['disk_available'] spec_ram = instance_type['memory_mb'] spec_disk = instance_type['local_gb'] - if host_ram_mb >= spec_ram and disk_bytes >= spec_disk: - selected_hosts.append((host, capabilities)) + extra_specs = instance_type['extra_specs'] + + if host_ram_mb >= spec_ram and \ + disk_bytes >= spec_disk and \ + self._satisfies_extra_specs(capabilities, instance_type): + selected_hosts.append((host, capabilities)) return selected_hosts #host entries (currently) are like: diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py index 10eafde0..75a2cb21 100644 --- a/nova/tests/scheduler/test_host_filter.py +++ b/nova/tests/scheduler/test_host_filter.py @@ -67,13 +67,24 @@ class HostFilterTestCase(test.TestCase): flavorid=1, swap=500, rxtx_quota=30000, - rxtx_cap=200) + rxtx_cap=200, + extra_specs={}) + self.gpu_instance_type = dict(name='tiny.gpu', + memory_mb=50, + vcpus=10, + local_gb=500, + flavorid=2, + swap=500, + rxtx_quota=30000, + rxtx_cap=200, + extra_specs={'gpu': 'nvidia'}) self.zone_manager = FakeZoneManager() states = {} for x in xrange(10): states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} self.zone_manager.service_states = states + self.zone_manager.service_states['host07']['compute']['gpu'] = 'nvidia' def tearDown(self): FLAGS.default_host_filter = self.old_flag @@ -116,6 +127,17 @@ class HostFilterTestCase(test.TestCase): self.assertEquals('host05', just_hosts[0]) self.assertEquals('host10', just_hosts[5]) + def test_instance_type_filter_extra_specs(self): + hf = host_filter.InstanceTypeFilter() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = hf.instance_type_to_filter(self.gpu_instance_type) + self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', + name) + hosts = hf.filter_hosts(self.zone_manager, cooked) + self.assertEquals(1, len(hosts)) + just_hosts = [host for host, caps in hosts] + self.assertEquals('host07', just_hosts[0]) + def test_json_filter(self): hf = host_filter.JsonFilter() # filter all hosts that can support 50 ram and 500 disk From 2ee0bad326aa8f3150de573b52e4bb29a0a2ea82 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 24 Jun 2011 01:44:22 -0400 Subject: [PATCH 27/40] Dealing with cases where extra_specs wasn't defined --- nova/scheduler/host_filter.py | 4 ++++ nova/tests/test_host_filter.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 7d4b9ad9..9c396cc0 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -97,9 +97,13 @@ class InstanceTypeFilter(HostFilter): """Check that the capabilities provided by the compute service satisfy the extra specs associated with the instance type""" + if 'extra_specs' not in instance_type: + return True + # Note(lorinh): For now, we are just checking exact matching on the # values. Later on, we want to handle numerical # values so we can represent things like number of GPU cards + try: for key, value in instance_type['extra_specs'].iteritems(): if capabilities[key] != value: diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py index 3361c7b7..438f3e52 100644 --- a/nova/tests/test_host_filter.py +++ b/nova/tests/test_host_filter.py @@ -67,7 +67,8 @@ class HostFilterTestCase(test.TestCase): flavorid=1, swap=500, rxtx_quota=30000, - rxtx_cap=200) + rxtx_cap=200, + extra_specs={}) self.zone_manager = FakeZoneManager() states = {} From 5bdfaf95ff450ff97326e3c04e7f0b2eec11f931 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 24 Jun 2011 15:20:24 +0400 Subject: [PATCH 28/40] Add reconnect on server fail to LDAP driver. --- nova/auth/ldapdriver.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index e9532473..4af91b61 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -101,6 +101,41 @@ def sanitize(fn): return _wrapped +class LDAPWrapper(object): + def __init__(self, ldap, url, user, password): + self.ldap = ldap + self.url = url + self.user = user + self.password = password + self.conn = None + + def __wrap_reconnect(f): + def inner(self, *args, **kwargs): + if self.conn is None: + self.connect() + return f(self.conn)(*args, **kwargs) + else: + try: + return f(self.conn)(*args, **kwargs) + except self.ldap.SERVER_DOWN: + self.connect() + return f(self.conn)(*args, **kwargs) + return inner + + def connect(self): + try: + self.conn = self.ldap.initialize(self.url) + self.conn.bind_s(self.user, self.password) + except self.ldap.SERVER_DOWN: + self.conn = None + raise + + search_s = __wrap_reconnect(lambda conn: conn.search_s) + add_s = __wrap_reconnect(lambda conn: conn.add_s) + delete_s = __wrap_reconnect(lambda conn: conn.delete_s) + modify_s = __wrap_reconnect(lambda conn: conn.modify_s) + + class LdapDriver(object): """Ldap Auth driver @@ -124,8 +159,8 @@ class LdapDriver(object): LdapDriver.project_objectclass = 'novaProject' self.__cache = None if LdapDriver.conn is None: - LdapDriver.conn = self.ldap.initialize(FLAGS.ldap_url) - LdapDriver.conn.simple_bind_s(FLAGS.ldap_user_dn, + LdapDriver.conn = LDAPWrapper(self.ldap, FLAGS.ldap_url, + FLAGS.ldap_user_dn, FLAGS.ldap_password) if LdapDriver.mc is None: LdapDriver.mc = memcache.Client(FLAGS.memcached_servers, debug=0) From 4f6cdd7f52a581a500d045db401f7f621b939074 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 24 Jun 2011 15:26:15 +0400 Subject: [PATCH 29/40] Use simple_bind_s instead of bind_s --- nova/auth/ldapdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 4af91b61..bc37d2d8 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -125,7 +125,7 @@ class LDAPWrapper(object): def connect(self): try: self.conn = self.ldap.initialize(self.url) - self.conn.bind_s(self.user, self.password) + self.conn.simple_bind_s(self.user, self.password) except self.ldap.SERVER_DOWN: self.conn = None raise From 3259bd06655132d68aff8f7830f3fc0d9ca1e7f4 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Fri, 24 Jun 2011 15:55:06 +0400 Subject: [PATCH 30/40] Add reconnect test. --- nova/auth/fakeldap.py | 24 ++++++++++++++++++++++++ nova/tests/test_auth.py | 9 +++++++++ 2 files changed, 33 insertions(+) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 79afb910..f1e76927 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -100,6 +100,11 @@ class OBJECT_CLASS_VIOLATION(Exception): # pylint: disable=C0103 pass +class SERVER_DOWN(Exception): # pylint: disable=C0103 + """Duplicate exception class from real LDAP module.""" + pass + + def initialize(_uri): """Opens a fake connection with an LDAP server.""" return FakeLDAP() @@ -202,25 +207,38 @@ def _to_json(unencoded): return json.dumps(list(unencoded)) +server_fail = False + + class FakeLDAP(object): """Fake LDAP connection.""" def simple_bind_s(self, dn, password): """This method is ignored, but provided for compatibility.""" + if server_fail: + raise SERVER_DOWN pass def unbind_s(self): """This method is ignored, but provided for compatibility.""" + if server_fail: + raise SERVER_DOWN pass def add_s(self, dn, attr): """Add an object with the specified attributes at dn.""" + if server_fail: + raise SERVER_DOWN + key = "%s%s" % (self.__prefix, dn) value_dict = dict([(k, _to_json(v)) for k, v in attr]) Store.instance().hmset(key, value_dict) def delete_s(self, dn): """Remove the ldap object at specified dn.""" + if server_fail: + raise SERVER_DOWN + Store.instance().delete("%s%s" % (self.__prefix, dn)) def modify_s(self, dn, attrs): @@ -232,6 +250,9 @@ class FakeLDAP(object): ([MOD_ADD | MOD_DELETE | MOD_REPACE], attribute, value) """ + if server_fail: + raise SERVER_DOWN + store = Store.instance() key = "%s%s" % (self.__prefix, dn) @@ -255,6 +276,9 @@ class FakeLDAP(object): fields -- fields to return. Returns all fields if not specified """ + if server_fail: + raise SERVER_DOWN + if scope != SCOPE_BASE and scope != SCOPE_SUBTREE: raise NotImplementedError(str(scope)) store = Store.instance() diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 7d00bddf..4aebbe94 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -25,6 +25,7 @@ from nova import log as logging from nova import test from nova.auth import manager from nova.api.ec2 import cloud +from nova.auth import fakeldap FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.auth_unittest') @@ -369,6 +370,14 @@ class _AuthManagerBaseTestCase(test.TestCase): class AuthManagerLdapTestCase(_AuthManagerBaseTestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' + def test_reconnect_on_server_failure(self): + self.manager.get_users() + fakeldap.server_fail = True + with self.assertRaises(fakeldap.SERVER_DOWN): + self.manager.get_users() + fakeldap.server_fail = False + self.manager.get_users() + class AuthManagerDbTestCase(_AuthManagerBaseTestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' From fa350ffee23822641baf5da58b8ec8e88ecb19a6 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 24 Jun 2011 10:52:59 -0400 Subject: [PATCH 31/40] Edited the host filter test case for extra specs --- nova/scheduler/host_filter.py | 4 ++-- nova/tests/scheduler/test_host_filter.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 9c396cc0..5f61f945 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -103,7 +103,7 @@ class InstanceTypeFilter(HostFilter): # Note(lorinh): For now, we are just checking exact matching on the # values. Later on, we want to handle numerical # values so we can represent things like number of GPU cards - + try: for key, value in instance_type['extra_specs'].iteritems(): if capabilities[key] != value: @@ -128,7 +128,7 @@ class InstanceTypeFilter(HostFilter): if host_ram_mb >= spec_ram and \ disk_bytes >= spec_disk and \ self._satisfies_extra_specs(capabilities, instance_type): - selected_hosts.append((host, capabilities)) + selected_hosts.append((host, capabilities)) return selected_hosts #host entries (currently) are like: diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py index 75a2cb21..b1892dab 100644 --- a/nova/tests/scheduler/test_host_filter.py +++ b/nova/tests/scheduler/test_host_filter.py @@ -77,14 +77,26 @@ class HostFilterTestCase(test.TestCase): swap=500, rxtx_quota=30000, rxtx_cap=200, - extra_specs={'gpu': 'nvidia'}) + extra_specs={'xpu_arch': 'fermi', + 'xpu_info': 'Tesla 2050'}) self.zone_manager = FakeZoneManager() states = {} for x in xrange(10): states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} self.zone_manager.service_states = states - self.zone_manager.service_states['host07']['compute']['gpu'] = 'nvidia' + + # Add some extra capabilities to some hosts + host07 = self.zone_manager.service_states['host07']['compute'] + host07['xpu_arch'] = 'fermi' + host07['xpu_info'] = 'Tesla 2050' + + host08 = self.zone_manager.service_states['host08']['compute'] + host08['xpu_arch'] = 'radeon' + + host09 = self.zone_manager.service_states['host09']['compute'] + host09['xpu_arch'] = 'fermi' + host09['xpu_info'] = 'Tesla 2150' def tearDown(self): FLAGS.default_host_filter = self.old_flag From 3ee2162cdd717a8facc9de35eafaae5a099dddf6 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Fri, 24 Jun 2011 23:55:18 +0400 Subject: [PATCH 33/40] stub tests --- Authors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Authors b/Authors index 94fcf7f5..bf2ba81d 100644 --- a/Authors +++ b/Authors @@ -29,7 +29,7 @@ Ewan Mellor Gabe Westmaas Hisaharu Ishii Hisaki Ohara -Ilya Alekseyev +Ilya Alekseyev Isaku Yamahata Jason Cannavale Jason Koelker From 7c9545f919b8b751f3a490b702750434a3223354 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Sat, 25 Jun 2011 00:33:40 +0400 Subject: [PATCH 34/40] some tests --- Authors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Authors b/Authors index bf2ba81d..09064051 100644 --- a/Authors +++ b/Authors @@ -29,7 +29,7 @@ Ewan Mellor Gabe Westmaas Hisaharu Ishii Hisaki Ohara -Ilya Alekseyev +Ilya Alekseyev Isaku Yamahata Jason Cannavale Jason Koelker From c00e3bcad01a740a0bf58536c65e1d5ff76e45f1 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Sat, 25 Jun 2011 03:03:17 +0400 Subject: [PATCH 35/40] fixes --- Authors | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Authors b/Authors index 09064051..c4281ca1 100644 --- a/Authors +++ b/Authors @@ -22,14 +22,14 @@ David Pravec Dean Troyer Devin Carlen Ed Leafe -Eldar Nugaev +Eldar Nugaev Eric Day Eric Windisch Ewan Mellor Gabe Westmaas Hisaharu Ishii Hisaki Ohara -Ilya Alekseyev +Ilya Alekseyev Isaku Yamahata Jason Cannavale Jason Koelker @@ -53,6 +53,7 @@ Kei Masumoto Ken Pepple Kevin Bringard Kevin L. Mitchell +Kirill Shileev Koji Iida Lorin Hochstein Lvov Maxim From d1efb8058ca1c747262cfc7fcef42bb316d3a50c Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Sat, 25 Jun 2011 04:45:15 +0400 Subject: [PATCH 36/40] mailmap --- .mailmap | 2 ++ Authors | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 3f0238ee..89094fbb 100644 --- a/.mailmap +++ b/.mailmap @@ -47,3 +47,5 @@ + + diff --git a/Authors b/Authors index c4281ca1..21a9bf94 100644 --- a/Authors +++ b/Authors @@ -29,7 +29,7 @@ Ewan Mellor Gabe Westmaas Hisaharu Ishii Hisaki Ohara -Ilya Alekseyev +Ilya Alekseyev Isaku Yamahata Jason Cannavale Jason Koelker From 8fce7d2f1696d936992c3bf7dfcbb28f75cf5039 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Sat, 25 Jun 2011 04:47:06 +0400 Subject: [PATCH 37/40] mailmap --- .mailmap | 2 ++ Authors | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.mailmap b/.mailmap index 89094fbb..6673d0a2 100644 --- a/.mailmap +++ b/.mailmap @@ -49,3 +49,5 @@ + + \ No newline at end of file diff --git a/Authors b/Authors index 21a9bf94..c3a65f1b 100644 --- a/Authors +++ b/Authors @@ -22,7 +22,7 @@ David Pravec Dean Troyer Devin Carlen Ed Leafe -Eldar Nugaev +Eldar Nugaev Eric Day Eric Windisch Ewan Mellor @@ -53,7 +53,7 @@ Kei Masumoto Ken Pepple Kevin Bringard Kevin L. Mitchell -Kirill Shileev +Kirill Shileev Koji Iida Lorin Hochstein Lvov Maxim From a17300baccb085b4e46dd24cf620f727a8d8008d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 25 Jun 2011 17:26:38 -0700 Subject: [PATCH 38/40] only create the db if it doesn't exist, add an option -r to run_tests.py to delete it --- run_tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/run_tests.py b/run_tests.py index bb33f913..fd836967 100644 --- a/run_tests.py +++ b/run_tests.py @@ -69,7 +69,6 @@ from nose import core from nose import result from nova import log as logging -from nova.tests import fake_flags class _AnsiColorizer(object): @@ -211,11 +210,11 @@ class NovaTestResult(result.TextTestResult): break sys.stdout = stdout - # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate - # error results in it failing to be initialized later. Otherwise, + # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate + # error results in it failing to be initialized later. Otherwise, # _handleElapsedTime will fail, causing the wrong error message to # be outputted. - self.start_time = time.time() + self.start_time = time.time() def getDescription(self, test): return str(test) From a2f08bf148f00dab5939c7682df33eb654787ffb Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 28 Jun 2011 18:52:22 +0400 Subject: [PATCH 40/40] Prevent test case from ruining other tests. Make it work in earlier python versions. --- nova/tests/test_auth.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 4aebbe94..71e0d17c 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -373,9 +373,10 @@ class AuthManagerLdapTestCase(_AuthManagerBaseTestCase): def test_reconnect_on_server_failure(self): self.manager.get_users() fakeldap.server_fail = True - with self.assertRaises(fakeldap.SERVER_DOWN): - self.manager.get_users() - fakeldap.server_fail = False + try: + self.assertRaises(fakeldap.SERVER_DOWN, self.manager.get_users) + finally: + fakeldap.server_fail = False self.manager.get_users()