From ed59ad68397acc14f5fb26e7e3ff6554b52f92a8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 15:06:42 -0700 Subject: [PATCH 01/90] update tests --- nova/tests/virt_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index ce78d450..d49383fb 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -91,7 +91,7 @@ class LibvirtConnTestCase(test.TrialTestCase): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) - uri, template = conn.get_uri_and_template() + uri, _template, _rescue = conn.get_uri_and_templates() self.assertEquals(uri, expected_uri) xml = conn.to_xml(instance_ref) @@ -114,7 +114,7 @@ class LibvirtConnTestCase(test.TrialTestCase): for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) - uri, template = conn.get_uri_and_template() + uri, _template, _rescue = conn.get_uri_and_templates() self.assertEquals(uri, testuri) def tearDown(self): From f6b6f48ff4a70af159e645ca26d8f6894c25413e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 00:45:33 -0700 Subject: [PATCH 02/90] ISCSI Volume support * Rewrite of Volume code to make VolumeManager more generic * AoE vs. iscsi moved to driver layer * Added db support for target ids * Added code to re-export volumes on restart of VolumeManager * Includes a script to create /dev/iscsi volumes on remote hosts --- nova/flags.py | 2 +- nova/tests/fake_flags.py | 8 +++++--- nova/tests/volume_unittest.py | 26 ++++++++++++-------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index f3b0384a..380382a7 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -224,7 +224,7 @@ DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager', 'Manager for compute') DEFINE_string('network_manager', 'nova.network.manager.VlanManager', 'Manager for network') -DEFINE_string('volume_manager', 'nova.volume.manager.AOEManager', +DEFINE_string('volume_manager', 'nova.volume.manager.VolumeManager', 'Manager for volume') DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 4bbef883..d695d68a 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -21,7 +21,7 @@ from nova import flags FLAGS = flags.FLAGS flags.DECLARE('volume_driver', 'nova.volume.manager') -FLAGS.volume_driver = 'nova.volume.driver.FakeAOEDriver' +FLAGS.volume_driver = 'nova.volume.driver.FakeISCSIDriver' FLAGS.connection_type = 'fake' FLAGS.fake_rabbit = True FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' @@ -31,9 +31,11 @@ flags.DECLARE('fake_network', 'nova.network.manager') FLAGS.network_size = 16 FLAGS.num_networks = 5 FLAGS.fake_network = True -flags.DECLARE('num_shelves', 'nova.volume.manager') -flags.DECLARE('blades_per_shelf', 'nova.volume.manager') +flags.DECLARE('num_shelves', 'nova.volume.driver') +flags.DECLARE('blades_per_shelf', 'nova.volume.driver') +flags.DECLARE('iscsi_target_ids', 'nova.volume.driver') FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 +FLAGS.iscsi_target_ids = 8 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index fdee30b4..34e04c8b 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -83,9 +83,9 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_many_volumes(self): - """Ensure that NoMoreBlades is raised when we run out of volumes""" + """Ensure that NoMoreTargets is raised when we run out of volumes""" vols = [] - total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf + total_slots = FLAGS.iscsi_target_ids for _index in xrange(total_slots): volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) @@ -93,7 +93,7 @@ class VolumeTestCase(test.TrialTestCase): volume_id = self._create_volume() self.assertFailure(self.volume.create_volume(self.context, volume_id), - db.NoMoreBlades) + db.NoMoreTargets) db.volume_destroy(context.get_admin_context(), volume_id) for volume_id in vols: yield self.volume.delete_volume(self.context, volume_id) @@ -148,23 +148,21 @@ class VolumeTestCase(test.TrialTestCase): db.instance_destroy(self.context, instance_id) @defer.inlineCallbacks - def test_concurrent_volumes_get_different_blades(self): - """Ensure multiple concurrent volumes get different blades""" + def test_concurrent_volumes_get_different_targets(self): + """Ensure multiple concurrent volumes get different targets""" volume_ids = [] - shelf_blades = [] + targets = [] def _check(volume_id): - """Make sure blades aren't duplicated""" + """Make sure targets aren't duplicated""" volume_ids.append(volume_id) admin_context = context.get_admin_context() - (shelf_id, blade_id) = db.volume_get_shelf_and_blade(admin_context, - volume_id) - shelf_blade = '%s.%s' % (shelf_id, blade_id) - self.assert_(shelf_blade not in shelf_blades) - shelf_blades.append(shelf_blade) - logging.debug("Blade %s allocated", shelf_blade) + target_id = db.volume_get_target_id(admin_context, volume_id) + self.assert_(target_id not in targets) + targets.append(target_id) + logging.debug("Target %s allocated", target_id) deferreds = [] - total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf + total_slots = FLAGS.iscsi_target_ids for _index in xrange(total_slots): volume_id = self._create_volume() d = self.volume.create_volume(self.context, volume_id) From c5bbe409dd622a92dc6bff637a13a7ea8a2cc43d Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 25 Oct 2010 12:25:50 -0400 Subject: [PATCH 03/90] pep8 --- nova/tests/api_unittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 80493b10..0b1c3e35 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -242,7 +242,7 @@ class ApiEc2TestCase(test.BaseTestCase): self.assertEquals(int(group.rules[0].from_port), 80) self.assertEquals(int(group.rules[0].to_port), 81) self.assertEquals(len(group.rules[0].grants), 1) - self.assertEquals(str(group.rules[0].grants[0]), '0.0.0.0/0') + self.assertEquals(str(group.rules[0].grants[0]), '0.0.0.0/0') self.expect_http() self.mox.ReplayAll() From 24807f08e32706bc79a361e7e031ddf41237aaa1 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Mon, 25 Oct 2010 22:42:49 +0000 Subject: [PATCH 04/90] Moving the openldap schema out of nova.sh into it's own files, and adding sun (opends/opendj/sun directory server/fedora ds) schema files --- nova/auth/nova_openldap.schema | 84 +++++++++++++++++++ nova/auth/nova_sun.schema | 16 ++++ nova/auth/openssh-lpk_openldap.schema | 19 +++++ nova/auth/openssh-lpk_sun.schema | 3 + nova/auth/slap.sh | 112 +------------------------- 5 files changed, 125 insertions(+), 109 deletions(-) create mode 100644 nova/auth/nova_openldap.schema create mode 100644 nova/auth/nova_sun.schema create mode 100644 nova/auth/openssh-lpk_openldap.schema create mode 100644 nova/auth/openssh-lpk_sun.schema diff --git a/nova/auth/nova_openldap.schema b/nova/auth/nova_openldap.schema new file mode 100644 index 00000000..4047361d --- /dev/null +++ b/nova/auth/nova_openldap.schema @@ -0,0 +1,84 @@ +# +# Person object for Nova +# inetorgperson with extra attributes +# Author: Vishvananda Ishaya +# +# + +# using internet experimental oid arc as per BP64 3.1 +objectidentifier novaSchema 1.3.6.1.3.1.666.666 +objectidentifier novaAttrs novaSchema:3 +objectidentifier novaOCs novaSchema:4 + +attributetype ( + novaAttrs:1 + NAME 'accessKey' + DESC 'Key for accessing data' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:2 + NAME 'secretKey' + DESC 'Secret key' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:3 + NAME 'keyFingerprint' + DESC 'Fingerprint of private key' + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:4 + NAME 'isAdmin' + DESC 'Is user an administrator?' + EQUALITY booleanMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 + SINGLE-VALUE + ) + +attributetype ( + novaAttrs:5 + NAME 'projectManager' + DESC 'Project Managers of a project' + SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 + ) + +objectClass ( + novaOCs:1 + NAME 'novaUser' + DESC 'access and secret keys' + AUXILIARY + MUST ( uid ) + MAY ( accessKey $ secretKey $ isAdmin ) + ) + +objectClass ( + novaOCs:2 + NAME 'novaKeyPair' + DESC 'Key pair for User' + SUP top + STRUCTURAL + MUST ( cn $ sshPublicKey $ keyFingerprint ) + ) + +objectClass ( + novaOCs:3 + NAME 'novaProject' + DESC 'Container for project' + SUP groupOfNames + STRUCTURAL + MUST ( cn $ projectManager ) + ) diff --git a/nova/auth/nova_sun.schema b/nova/auth/nova_sun.schema new file mode 100644 index 00000000..e925e05e --- /dev/null +++ b/nova/auth/nova_sun.schema @@ -0,0 +1,16 @@ +# +# Person object for Nova +# inetorgperson with extra attributes +# Author: Vishvananda Ishaya +# Modified for strict RFC 4512 compatibility by: Ryan Lane +# +# using internet experimental oid arc as per BP64 3.1 +dn: cn=schema +attributeTypes: ( 1.3.6.1.3.1.666.666.3.1 NAME 'accessKey' DESC 'Key for accessing data' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.2 NAME 'secretKey' DESC 'Secret key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.3 NAME 'keyFingerprint' DESC 'Fingerprint of private key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.4 NAME 'isAdmin' DESC 'Is user an administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.5 NAME 'projectManager' DESC 'Project Managers of a project' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 ) +objectClasses: ( 1.3.6.1.3.1.666.666.4.1 NAME 'novaUser' DESC 'access and secret keys' SUP top AUXILIARY MUST ( uid ) MAY ( accessKey $ secretKey $ isAdmin ) ) +objectClasses: ( 1.3.6.1.3.1.666.666.4.2 NAME 'novaKeyPair' DESC 'Key pair for User' SUP top STRUCTURAL MUST ( cn $ sshPublicKey $ keyFingerprint ) ) +objectClasses: ( 1.3.6.1.3.1.666.666.4.3 NAME 'novaProject' DESC 'Container for project' SUP groupOfNames STRUCTURAL MUST ( cn $ projectManager ) ) diff --git a/nova/auth/openssh-lpk_openldap.schema b/nova/auth/openssh-lpk_openldap.schema new file mode 100644 index 00000000..93351da6 --- /dev/null +++ b/nova/auth/openssh-lpk_openldap.schema @@ -0,0 +1,19 @@ +# +# LDAP Public Key Patch schema for use with openssh-ldappubkey +# Author: Eric AUGE +# +# Based on the proposal of : Mark Ruijter +# + + +# octetString SYNTAX +attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' + DESC 'MANDATORY: OpenSSH Public key' + EQUALITY octetStringMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) + +# printableString SYNTAX yes|no +objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY + DESC 'MANDATORY: OpenSSH LPK objectclass' + MAY ( sshPublicKey $ uid ) + ) diff --git a/nova/auth/openssh-lpk_sun.schema b/nova/auth/openssh-lpk_sun.schema new file mode 100644 index 00000000..5b220ab0 --- /dev/null +++ b/nova/auth/openssh-lpk_sun.schema @@ -0,0 +1,3 @@ +dn: cn=schema +attributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' DESC 'MANDATORY: OpenSSH Public key' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) +objectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY DESC 'MANDATORY: OpenSSH LPK objectclass' MAY ( sshPublicKey $ uid ) ) diff --git a/nova/auth/slap.sh b/nova/auth/slap.sh index fdc0e39d..797675d2 100755 --- a/nova/auth/slap.sh +++ b/nova/auth/slap.sh @@ -20,115 +20,9 @@ apt-get install -y slapd ldap-utils python-ldap -cat >/etc/ldap/schema/openssh-lpk_openldap.schema < -# -# Based on the proposal of : Mark Ruijter -# - - -# octetString SYNTAX -attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' - DESC 'MANDATORY: OpenSSH Public key' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -# printableString SYNTAX yes|no -objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY - DESC 'MANDATORY: OpenSSH LPK objectclass' - MAY ( sshPublicKey $ uid ) - ) -LPK_SCHEMA_EOF - -cat >/etc/ldap/schema/nova.schema < -# -# - -# using internet experimental oid arc as per BP64 3.1 -objectidentifier novaSchema 1.3.6.1.3.1.666.666 -objectidentifier novaAttrs novaSchema:3 -objectidentifier novaOCs novaSchema:4 - -attributetype ( - novaAttrs:1 - NAME 'accessKey' - DESC 'Key for accessing data' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:2 - NAME 'secretKey' - DESC 'Secret key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:3 - NAME 'keyFingerprint' - DESC 'Fingerprint of private key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:4 - NAME 'isAdmin' - DESC 'Is user an administrator?' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:5 - NAME 'projectManager' - DESC 'Project Managers of a project' - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 - ) - -objectClass ( - novaOCs:1 - NAME 'novaUser' - DESC 'access and secret keys' - AUXILIARY - MUST ( uid ) - MAY ( accessKey $ secretKey $ isAdmin ) - ) - -objectClass ( - novaOCs:2 - NAME 'novaKeyPair' - DESC 'Key pair for User' - SUP top - STRUCTURAL - MUST ( cn $ sshPublicKey $ keyFingerprint ) - ) - -objectClass ( - novaOCs:3 - NAME 'novaProject' - DESC 'Container for project' - SUP groupOfNames - STRUCTURAL - MUST ( cn $ projectManager ) - ) - -NOVA_SCHEMA_EOF +abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` +cp $abspath/openssh-lpk_openldap.schema /etc/ldap/schema/openssh-lpk_openldap.schema +cp $abspath/nova_openldap.schema /etc/ldap/schema/nova_openldap.schema mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig cat >/etc/ldap/slapd.conf < Date: Mon, 25 Oct 2010 22:50:32 +0000 Subject: [PATCH 05/90] Documentation was missing; added --- nova/auth/openssh-lpk_sun.schema | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nova/auth/openssh-lpk_sun.schema b/nova/auth/openssh-lpk_sun.schema index 5b220ab0..5f52db3b 100644 --- a/nova/auth/openssh-lpk_sun.schema +++ b/nova/auth/openssh-lpk_sun.schema @@ -1,3 +1,10 @@ +# +# LDAP Public Key Patch schema for use with openssh-ldappubkey +# Author: Eric AUGE +# +# Schema for Sun Directory Server. +# Based on the original schema, modified by Stefan Fischer. +# dn: cn=schema attributeTypes: ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' DESC 'MANDATORY: OpenSSH Public key' EQUALITY octetStringMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) objectClasses: ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY DESC 'MANDATORY: OpenSSH LPK objectclass' MAY ( sshPublicKey $ uid ) ) From 1e053da8cfc1fe163253edb91974a87950e9269e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 23:04:49 -0700 Subject: [PATCH 06/90] fix completely broken ServiceTestCase --- nova/tests/service_unittest.py | 125 ++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 55 deletions(-) diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index e74e0f72..a268bc4f 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -23,8 +23,8 @@ Unit Tests for remote procedure calls using queue import mox from twisted.application.app import startApplication +from twisted.internet import defer -from nova import context from nova import exception from nova import flags from nova import rpc @@ -48,7 +48,7 @@ class ExtendedService(service.Service): return 'service' -class ServiceManagerTestCase(test.BaseTestCase): +class ServiceManagerTestCase(test.TrialTestCase): """Test cases for Services""" def test_attribute_error_for_no_manager(self): @@ -75,13 +75,12 @@ class ServiceManagerTestCase(test.BaseTestCase): self.assertEqual(serv.test_method(), 'service') -class ServiceTestCase(test.BaseTestCase): +class ServiceTestCase(test.TrialTestCase): """Test cases for Services""" def setUp(self): super(ServiceTestCase, self).setUp() self.mox.StubOutWithMock(service, 'db') - self.context = context.get_admin_context() def test_create(self): host = 'foo' @@ -144,87 +143,103 @@ class ServiceTestCase(test.BaseTestCase): # whether it is disconnected, it looks for a variable on itself called # 'model_disconnected' and report_state doesn't really do much so this # these are mostly just for coverage - def test_report_state(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - + @defer.inlineCallbacks def test_report_state_no_service(self): host = 'foo' binary = 'bar' + topic = 'test' service_create = {'host': host, 'binary': binary, + 'topic': topic, 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, + service.db.service_get_by_args(mox.IgnoreArg(), host, binary).AndRaise(exception.NotFound()) - service.db.service_create(self.context, + service.db.service_create(mox.IgnoreArg(), service_create).AndReturn(service_ref) - service.db.service_get(self.context, + service.db.service_get(mox.IgnoreArg(), service_ref['id']).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], + service.db.service_update(mox.IgnoreArg(), service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + yield serv.report_state() + @defer.inlineCallbacks def test_report_state_newly_disconnected(self): host = 'foo' binary = 'bar' + topic = 'test' + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(Exception()) + service.db.service_get_by_args(mox.IgnoreArg(), + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(mox.IgnoreArg(), + service_create).AndReturn(service_ref) + service.db.service_get(mox.IgnoreArg(), + mox.IgnoreArg()).AndRaise(Exception()) self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - self.assert_(s.model_disconnected) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + yield serv.report_state() + self.assert_(serv.model_disconnected) + @defer.inlineCallbacks def test_report_state_newly_connected(self): host = 'foo' binary = 'bar' + topic = 'test' + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], + service.db.service_get_by_args(mox.IgnoreArg(), + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(mox.IgnoreArg(), + service_create).AndReturn(service_ref) + service.db.service_get(mox.IgnoreArg(), + service_ref['id']).AndReturn(service_ref) + service.db.service_update(mox.IgnoreArg(), service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() - s = service.Service() - s.model_disconnected = True - rv = yield s.report_state(host, binary) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + serv.model_disconnected = True + yield serv.report_state() - self.assert_(not s.model_disconnected) + self.assert_(not serv.model_disconnected) From 4d75a82cb29486121cd68c43d6c8e64de09430ee Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 26 Oct 2010 11:48:20 -0400 Subject: [PATCH 07/90] Delete BaseTestCase and with it the last reference to tornado. Requires commenting out some service_unittest tests which were silently failing under BaseTestCase and which now fail under TrialTestCase. vishy says he wrote the code and thinks he knows what is going wrong. --- nova/tests/api_unittest.py | 4 +- nova/tests/service_unittest.py | 184 +++++++++++++++++---------------- run_tests.py | 16 --- 3 files changed, 96 insertions(+), 108 deletions(-) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 0b1c3e35..0a81c575 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -83,7 +83,7 @@ class FakeHttplibConnection(object): pass -class XmlConversionTestCase(test.BaseTestCase): +class XmlConversionTestCase(test.TrialTestCase): """Unit test api xml conversion""" def test_number_conversion(self): conv = apirequest._try_convert @@ -100,7 +100,7 @@ class XmlConversionTestCase(test.BaseTestCase): self.assertEqual(conv('-0'), 0) -class ApiEc2TestCase(test.BaseTestCase): +class ApiEc2TestCase(test.TrialTestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): super(ApiEc2TestCase, self).setUp() diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index e74e0f72..142c2ebe 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -48,7 +48,7 @@ class ExtendedService(service.Service): return 'service' -class ServiceManagerTestCase(test.BaseTestCase): +class ServiceManagerTestCase(test.TrialTestCase): """Test cases for Services""" def test_attribute_error_for_no_manager(self): @@ -75,7 +75,7 @@ class ServiceManagerTestCase(test.BaseTestCase): self.assertEqual(serv.test_method(), 'service') -class ServiceTestCase(test.BaseTestCase): +class ServiceTestCase(test.TrialTestCase): """Test cases for Services""" def setUp(self): @@ -140,91 +140,95 @@ class ServiceTestCase(test.BaseTestCase): startApplication(app, False) self.assert_(app) - # We're testing sort of weird behavior in how report_state decides - # whether it is disconnected, it looks for a variable on itself called - # 'model_disconnected' and report_state doesn't really do much so this - # these are mostly just for coverage - def test_report_state(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - def test_report_state_no_service(self): - host = 'foo' - binary = 'bar' - service_create = {'host': host, - 'binary': binary, - 'report_count': 0} - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(self.context, - service_create).AndReturn(service_ref) - service.db.service_get(self.context, - service_ref['id']).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - def test_report_state_newly_disconnected(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(Exception()) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - self.assert_(s.model_disconnected) - - def test_report_state_newly_connected(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - s.model_disconnected = True - rv = yield s.report_state(host, binary) - - self.assert_(not s.model_disconnected) +# TODO(gundlach): These tests were "passing" when this class inherited from +# BaseTestCase. In reality, they were failing, but BaseTestCase was +# swallowing the error. Now that we inherit from TrialTestCase, these tests +# are failing, and need to get fixed. +# # We're testing sort of weird behavior in how report_state decides +# # whether it is disconnected, it looks for a variable on itself called +# # 'model_disconnected' and report_state doesn't really do much so this +# # these are mostly just for coverage +# def test_report_state(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# def test_report_state_no_service(self): +# host = 'foo' +# binary = 'bar' +# service_create = {'host': host, +# 'binary': binary, +# 'report_count': 0} +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndRaise(exception.NotFound()) +# service.db.service_create(self.context, +# service_create).AndReturn(service_ref) +# service.db.service_get(self.context, +# service_ref['id']).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# def test_report_state_newly_disconnected(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndRaise(Exception()) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# self.assert_(s.model_disconnected) +# +# def test_report_state_newly_connected(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# s.model_disconnected = True +# rv = yield s.report_state(host, binary) +# +# self.assert_(not s.model_disconnected) diff --git a/run_tests.py b/run_tests.py index 9a2f40dc..c16c6324 100644 --- a/run_tests.py +++ b/run_tests.py @@ -48,24 +48,8 @@ from twisted.scripts import trial as trial_script from nova import flags from nova import twistd -from nova.tests.access_unittest import * from nova.tests.auth_unittest import * -from nova.tests.api_unittest import * -from nova.tests.cloud_unittest import * -from nova.tests.compute_unittest import * -from nova.tests.flags_unittest import * -from nova.tests.network_unittest import * -from nova.tests.objectstore_unittest import * -from nova.tests.process_unittest import * -from nova.tests.quota_unittest import * -from nova.tests.rpc_unittest import * -from nova.tests.scheduler_unittest import * from nova.tests.service_unittest import * -from nova.tests.twistd_unittest import * -from nova.tests.validator_unittest import * -from nova.tests.virt_unittest import * -from nova.tests.volume_unittest import * -from nova.tests.virt_unittest import * FLAGS = flags.FLAGS From fa3fa4394ffcb3034424ad0c99eae32d8c8fdfad Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 26 Oct 2010 11:58:46 -0400 Subject: [PATCH 08/90] Oops, didn't mean to check this one in. Ninja-patch --- run_tests.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/run_tests.py b/run_tests.py index c16c6324..9a2f40dc 100644 --- a/run_tests.py +++ b/run_tests.py @@ -48,8 +48,24 @@ from twisted.scripts import trial as trial_script from nova import flags from nova import twistd +from nova.tests.access_unittest import * from nova.tests.auth_unittest import * +from nova.tests.api_unittest import * +from nova.tests.cloud_unittest import * +from nova.tests.compute_unittest import * +from nova.tests.flags_unittest import * +from nova.tests.network_unittest import * +from nova.tests.objectstore_unittest import * +from nova.tests.process_unittest import * +from nova.tests.quota_unittest import * +from nova.tests.rpc_unittest import * +from nova.tests.scheduler_unittest import * from nova.tests.service_unittest import * +from nova.tests.twistd_unittest import * +from nova.tests.validator_unittest import * +from nova.tests.virt_unittest import * +from nova.tests.volume_unittest import * +from nova.tests.virt_unittest import * FLAGS = flags.FLAGS From 803b042f4b4fba73a47d53fe6d8c813628a6ae74 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Wed, 27 Oct 2010 17:31:46 -0400 Subject: [PATCH 10/90] cleanup rrd doc generation. --- nova/auth/fakeldap.py | 1 - nova/auth/manager.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index cf3a84a5..1a49b73f 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -79,7 +79,6 @@ def _match_query(query, attrs): &, |, and ! are supported in the query. No syntax checking is performed, so malformed querys will not work correctly. - """ # cut off the parentheses inner = query[1:-1] diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 001a9687..7b2b6816 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -84,12 +84,11 @@ class AuthBase(object): @classmethod def safe_id(cls, obj): - """Safe get object id + """Safely get object id. This method will return the id of the object if the object is of this class, otherwise it will return the original object. This allows methods to accept objects or ids as paramaters. - """ if isinstance(obj, cls): return obj.id From d6da5499878ce3c68ac2df29b9c77b04e79fbad2 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 27 Oct 2010 16:33:25 -0700 Subject: [PATCH 11/90] updates to auth, concepts, and network, fix of docstring --- nova/auth/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 001a9687..8388d5a5 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -89,7 +89,6 @@ class AuthBase(object): This method will return the id of the object if the object is of this class, otherwise it will return the original object. This allows methods to accept objects or ids as paramaters. - """ if isinstance(obj, cls): return obj.id From 5cdca0dade5db60a38235fe48a0f35f45884493c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Thu, 28 Oct 2010 12:25:39 -0400 Subject: [PATCH 12/90] Whitespace and docstring cleanups --- nova/auth/fakeldap.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 1a49b73f..176a00f0 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -15,12 +15,14 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + """ -Fake LDAP server for test harnesses. +Fake LDAP server for test harness, backs to ReDIS. This class does very little error checking, and knows nothing about ldap -class definitions. It implements the minimum emulation of the python ldap +class definitions. It implements the minimum emulation of the python ldap library to work with nova. + """ import json @@ -77,8 +79,8 @@ def initialize(_uri): def _match_query(query, attrs): """Match an ldap query to an attribute dictionary. - &, |, and ! are supported in the query. No syntax checking is performed, - so malformed querys will not work correctly. + The characters &, |, and ! are supported in the query. No syntax checking + is performed, so malformed querys will not work correctly. """ # cut off the parentheses inner = query[1:-1] From 6dd1be7a2867b041133543d8384218e12c2cf8d2 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 11:58:57 -0400 Subject: [PATCH 13/90] Document Fakes --- nova/auth/fakeldap.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 176a00f0..46e0135b 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -15,9 +15,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -""" -Fake LDAP server for test harness, backs to ReDIS. +"""Fake LDAP server for test harness, backs to ReDIS. This class does very little error checking, and knows nothing about ldap class definitions. It implements the minimum emulation of the python ldap From 033f3167b77bd9d5d83ef563d658420c34f9026d Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 29 Oct 2010 16:18:00 -0400 Subject: [PATCH 15/90] Volume documentation. --- nova/tests/volume_unittest.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index fdee30b4..896800ce 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -16,7 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. """ -Tests for Volume Code +Tests for Volume Code. + """ import logging @@ -33,7 +34,8 @@ FLAGS = flags.FLAGS class VolumeTestCase(test.TrialTestCase): - """Test Case for volumes""" + """Test Case for volumes.""" + def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(VolumeTestCase, self).setUp() @@ -44,7 +46,7 @@ class VolumeTestCase(test.TrialTestCase): @staticmethod def _create_volume(size='0'): - """Create a volume object""" + """Create a volume object.""" vol = {} vol['size'] = size vol['user_id'] = 'fake' @@ -56,7 +58,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_create_delete_volume(self): - """Test volume can be created and deleted""" + """Test volume can be created and deleted.""" volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) self.assertEqual(volume_id, db.volume_get(context.get_admin_context(), @@ -70,7 +72,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_big_volume(self): - """Ensure failure if a too large of a volume is requested""" + """Ensure failure if a too large of a volume is requested.""" # FIXME(vish): validation needs to move into the data layer in # volume_create defer.returnValue(True) @@ -83,7 +85,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_too_many_volumes(self): - """Ensure that NoMoreBlades is raised when we run out of volumes""" + """Ensure that NoMoreBlades is raised when we run out of volumes.""" vols = [] total_slots = FLAGS.num_shelves * FLAGS.blades_per_shelf for _index in xrange(total_slots): @@ -100,7 +102,7 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_run_attach_detach_volume(self): - """Make sure volume can be attached and detached from instance""" + """Make sure volume can be attached and detached from instance.""" inst = {} inst['image_id'] = 'ami-test' inst['reservation_id'] = 'r-fakeres' @@ -149,12 +151,13 @@ class VolumeTestCase(test.TrialTestCase): @defer.inlineCallbacks def test_concurrent_volumes_get_different_blades(self): - """Ensure multiple concurrent volumes get different blades""" + """Ensure multiple concurrent volumes get different blades.""" + volume_ids = [] shelf_blades = [] def _check(volume_id): - """Make sure blades aren't duplicated""" + """Make sure blades aren't duplicated.""" volume_ids.append(volume_id) admin_context = context.get_admin_context() (shelf_id, blade_id) = db.volume_get_shelf_and_blade(admin_context, From 9a08a7399fb321477df63a2927e22005653be75b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 30 Oct 2010 20:57:18 -0700 Subject: [PATCH 16/90] Change retrieval of security groups from kwargs so they are associated properly and add test to verify --- nova/tests/compute_unittest.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 01b5651d..71a1a445 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -66,6 +66,27 @@ class ComputeTestCase(test.TrialTestCase): inst['ami_launch_index'] = 0 return db.instance_create(self.context, inst)['id'] + def test_create_instance_associates_security_groups(self): + """Make sure create_instance associates security groups""" + inst = {} + inst['user_id'] = self.user.id + inst['project_id'] = self.project.id + values = {'name': 'default', + 'description': 'default', + 'user_id': self.user.id, + 'project_id': self.project.id} + group = db.security_group_create(self.context, values) + ref = self.compute.create_instance(self.context, + security_groups=[group['id']], + **inst) + # reload to get groups + instance_ref = db.instance_get(self.context, ref['id']) + try: + self.assertEqual(len(instance_ref['security_groups']), 1) + finally: + db.security_group_destroy(self.context, group['id']) + db.instance_destroy(self.context, instance_ref['id']) + @defer.inlineCallbacks def test_run_terminate(self): """Make sure it is possible to run and terminate instance""" From b01bf759ed288a5f0b0f7fcc03ac5192ba2642fb Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 2 Nov 2010 11:28:14 -0700 Subject: [PATCH 17/90] Added support for OpenStack and EC2 APIs to run on different ports. --- bin/nova-api | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 20f1bd74..a9002ae2 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -37,13 +37,18 @@ from nova import utils from nova import server FLAGS = flags.FLAGS -flags.DEFINE_integer('api_port', 8773, 'API port') +flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') +flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') def main(_args): from nova import api from nova import wsgi - wsgi.run_server(api.API(), FLAGS.api_port) + server = wsgi.Server() + server.start(api.API('os'), FLAGS.osapi_port) + server.start(api.API('ec2'), FLAGS.ec2api_port) + server.wait() + if __name__ == '__main__': utils.default_flagfile() From 845d894f6deb5eab35267c940bb2ad81630822ab Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 2 Nov 2010 12:02:42 -0700 Subject: [PATCH 18/90] Fixed tests to work with new default API argument. --- nova/tests/api_unittest.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 0a81c575..33d4cb29 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -34,10 +34,6 @@ from nova.api.ec2 import apirequest from nova.auth import manager -FLAGS = flags.FLAGS -FLAGS.FAKE_subdomain = 'ec2' - - class FakeHttplibSocket(object): """a fake socket implementation for httplib.HTTPResponse, trivial""" def __init__(self, response_string): @@ -109,7 +105,7 @@ class ApiEc2TestCase(test.TrialTestCase): self.host = '127.0.0.1' - self.app = api.API() + self.app = api.API('ec2') def expect_http(self, host=None, is_secure=False): """Returns a new EC2 connection""" From 4c3178715242e664ab5ea6dcdd4f5b9975c8bfd6 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 2 Nov 2010 13:51:09 -0700 Subject: [PATCH 19/90] Fixed --help display for non-twisted bin/* commands. --- nova/flags.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/flags.py b/nova/flags.py index f3b0384a..e51f286a 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -138,6 +138,8 @@ class FlagValues(gflags.FlagValues): FLAGS = FlagValues() +gflags.FLAGS = FLAGS +gflags.DEFINE_flag(gflags.HelpFlag(), FLAGS) def _wrapper(func): From 940ffc11722cdae89ed77ad2a1b7456fd8cb7370 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 2 Nov 2010 20:31:17 -0400 Subject: [PATCH 20/90] Fixes after trunk merge. --- nova/tests/fake_flags.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 4bbef883..bc377f0d 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -24,6 +24,7 @@ flags.DECLARE('volume_driver', 'nova.volume.manager') FLAGS.volume_driver = 'nova.volume.driver.FakeAOEDriver' FLAGS.connection_type = 'fake' FLAGS.fake_rabbit = True +flags.DECLARE('auth_driver', 'nova.auth.manager') FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') From a7698f931eada80907c8ab68e2f05e41be9d40df Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 3 Nov 2010 12:38:15 -0700 Subject: [PATCH 21/90] Fix for bug#613264, allowing hosts to be specified for nova-api and objectstore listeners. --- bin/nova-api | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index a9002ae2..a9c53dbc 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -38,15 +38,17 @@ from nova import server FLAGS = flags.FLAGS flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') +flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') +flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') def main(_args): from nova import api from nova import wsgi server = wsgi.Server() - server.start(api.API('os'), FLAGS.osapi_port) - server.start(api.API('ec2'), FLAGS.ec2api_port) + server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) + server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) server.wait() From dccd1ee56785cb825f044e9f5db45dd84725c080 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 3 Nov 2010 14:59:35 -0700 Subject: [PATCH 23/90] make sure context keys are not unicode so they can be passed as kwargs --- nova/rpc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/rpc.py b/nova/rpc.py index 895820cd..05eaa0f9 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -262,6 +262,9 @@ def _unpack_context(msg): """Unpack context from msg.""" context_dict = {} for key in list(msg.keys()): + # NOTE(vish): Some versions of python don't like unicode keys + # in kwargs. + key = str(key) if key.startswith('_context_'): value = msg.pop(key) context_dict[key[9:]] = value From 08c99e649f0fd4f3e14faf6f574724164f9ebdb5 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 3 Nov 2010 15:06:00 -0700 Subject: [PATCH 24/90] pep8 whitespace and line length fixes --- nova/adminclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index 0227cddd..af55197f 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -273,10 +273,10 @@ class NovaAdminClient(object): def get_user_roles(self, user, project=None): """Returns a list of roles for the given user. - + Omitting project will return any global roles that the user has. Specifying project will return only project specific roles. - + """ params = {'User': user} if project: From 6403c61db7e0ca0403bbf1d3dc961af84143bf5e Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 3 Nov 2010 15:50:24 -0700 Subject: [PATCH 25/90] Fix for bug #640400, enables the exclusive flag on the temporary queues. --- nova/rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/rpc.py b/nova/rpc.py index 895820cd..14fe010a 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -206,6 +206,7 @@ class DirectConsumer(Consumer): self.routing_key = msg_id self.exchange = msg_id self.auto_delete = True + self.exclusive = True super(DirectConsumer, self).__init__(connection=connection) From 96946f80c9bc0980999c1cf8e0f27d9cb52929d5 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 4 Nov 2010 12:42:14 +0100 Subject: [PATCH 26/90] Add a templating mechanism in the flag parsing. Add a state_path flag that will be used as the top-level dir for all other state (such as images, instances, buckets, networks, etc). This way you only need to change one flag to put all your state in e.g. /var/lib/nova. --- nova/flags.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 4ae86d9b..2b8bbbdb 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -26,6 +26,8 @@ import os import socket import sys +from string import Template + import gflags @@ -134,8 +136,21 @@ class FlagValues(gflags.FlagValues): def __getattr__(self, name): if self.IsDirty(name): self.ParseNewFlags() - return gflags.FlagValues.__getattr__(self, name) + val = gflags.FlagValues.__getattr__(self, name) + if type(val) is str: + tmpl = Template(val) + return tmpl.substitute(StrWrapper(self)) + return val +class StrWrapper(object): + def __init__(self, obj): + self.wrapped = obj + + def __getitem__(self, name): + if hasattr(self.wrapped, name): + return str(getattr(self.wrapped, name)) + else: + raise KeyError(name) FLAGS = FlagValues() gflags.FLAGS = FLAGS @@ -218,8 +233,11 @@ DEFINE_string('vpn_key_suffix', DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') +DEFINE_string('state_path', os.path.abspath("./"), + "Top-level directory for maintaining nova's state") + DEFINE_string('sql_connection', - 'sqlite:///%s/nova.sqlite' % os.path.abspath("./"), + 'sqlite:///$state_path/nova.sqlite', 'connection string for sql database') DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager', From a98f268c7f7595086d2626232df1ccc26429c175 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 11 Nov 2010 19:52:36 -0600 Subject: [PATCH 31/90] Added a .mailmap that maps addresses in bzr to people's real, preferred e-mail addresses. (I made a few guesses along the way, feel free to adjust according to what is actually the preferred e-mail) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added a couple of methods to nova.utils to parse said .mailmap and do the appropriate (though highly naïve) replacement. Apply mailmap replacement in changelog generation in setup.py. Add a unit test that checks everyone is properly listed in Authors. Add sleepsonthefloor to Authors. If anyone knows the real name, please add it. --- .mailmap | 24 +++++++++++++++++++ Authors | 1 + nova/tests/misc_unittest.py | 48 +++++++++++++++++++++++++++++++++++++ run_tests.py | 5 ++-- 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 .mailmap create mode 100644 nova/tests/misc_unittest.py diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..cf79dc95 --- /dev/null +++ b/.mailmap @@ -0,0 +1,24 @@ +# Format is: +# + + + + + + + + + + + + + + + + + + + + + + diff --git a/Authors b/Authors index ec3a1cbd..87ebb55d 100644 --- a/Authors +++ b/Authors @@ -19,3 +19,4 @@ Rick Clark Soren Hansen Todd Willey Vishvananda Ishaya +¿¿¿??? diff --git a/nova/tests/misc_unittest.py b/nova/tests/misc_unittest.py new file mode 100644 index 00000000..856060af --- /dev/null +++ b/nova/tests/misc_unittest.py @@ -0,0 +1,48 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import subprocess + +from nova import test +from nova.utils import parse_mailmap, str_dict_replace + + +class ProjectTestCase(test.TrialTestCase): + def test_authors_up_to_date(self): + if os.path.exists('../.bzr'): + log_cmd = subprocess.Popen(["bzr", "log", "-n0"], + stdout=subprocess.PIPE) + changelog = log_cmd.communicate()[0] + mailmap = parse_mailmap('../.mailmap') + + contributors = set() + for l in changelog.split('\n'): + l = l.strip() + if (l.startswith('author:') or l.startswith('committer:') + and not l == 'committer: Tarmac'): + email = l.split(' ')[-1] + contributors.add(str_dict_replace(email, mailmap)) + + authors_file = open('../Authors', 'r').read() + + missing = set() + for contributor in contributors: + if not contributor in authors_file: + missing.add(contributor) + + self.assertTrue(len(missing) == 0, + '%r not listed in Authors' % missing) diff --git a/run_tests.py b/run_tests.py index 9a2f40dc..3d427d8a 100644 --- a/run_tests.py +++ b/run_tests.py @@ -49,11 +49,12 @@ from nova import flags from nova import twistd from nova.tests.access_unittest import * -from nova.tests.auth_unittest import * from nova.tests.api_unittest import * +from nova.tests.auth_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * from nova.tests.flags_unittest import * +from nova.tests.misc_unittest import * from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * from nova.tests.process_unittest import * @@ -64,8 +65,8 @@ from nova.tests.service_unittest import * from nova.tests.twistd_unittest import * from nova.tests.validator_unittest import * from nova.tests.virt_unittest import * -from nova.tests.volume_unittest import * from nova.tests.virt_unittest import * +from nova.tests.volume_unittest import * FLAGS = flags.FLAGS From ce5cd2d9567dd8023eee0f6af94bf8dc7082d5e8 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sat, 13 Nov 2010 09:59:07 -0800 Subject: [PATCH 33/90] removed redundant unit test import --- run_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index 9a2f40dc..101ed1a0 100644 --- a/run_tests.py +++ b/run_tests.py @@ -65,7 +65,6 @@ from nova.tests.twistd_unittest import * from nova.tests.validator_unittest import * from nova.tests.virt_unittest import * from nova.tests.volume_unittest import * -from nova.tests.virt_unittest import * FLAGS = flags.FLAGS From 2da2f90b578457d212978112ac015e37bd8d8550 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Mon, 15 Nov 2010 13:15:48 -0600 Subject: [PATCH 34/90] Adding contributors and names --- Authors | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Authors b/Authors index 87ebb55d..5b1d2903 100644 --- a/Authors +++ b/Authors @@ -1,5 +1,7 @@ Andy Smith Anne Gentle +Anthony Young +Armando Migliaccio Chris Behrens Devin Carlen Eric Day @@ -8,7 +10,7 @@ Hisaki Ohara Jay Pipes Jesse Andrews Joe Heck -Joel Moore joelbm24@gmail.com +Joel Moore Joshua McKenty Justin Santa Barbara Matt Dietz @@ -19,4 +21,5 @@ Rick Clark Soren Hansen Todd Willey Vishvananda Ishaya -¿¿¿??? +Youcef Laribi +Zhixue Wu From be1f175cfe8fe08b991a6d061e9ec43e598327f1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 16 Nov 2010 23:38:37 +0000 Subject: [PATCH 35/90] fixes errors in describe address and associate address. Adds test cases --- nova/tests/cloud_unittest.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 2d61d267..938210fa 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -91,6 +91,35 @@ class CloudTestCase(test.TrialTestCase): # NOTE(vish): create depends on pool, so just call helper directly return cloud._gen_key(self.context, self.context.user.id, name) + def test_describe_addresses(self): + """Makes sure describe addresses runs without raising an exception""" + address = "10.10.10.10" + db.floating_ip_create(context.get_admin_context(), + {'address': address, + 'host': FLAGS.host}) + self.cloud.allocate_address(self.context) + self.cloud.describe_addresses(self.context) + self.cloud.release_address(self.context, + public_ip=address) + + def test_associate_disassociate_address(self): + """Verifies associate runs cleanly without raising an exception""" + address = "10.10.10.10" + db.floating_ip_create(context.get_admin_context(), + {'address': address, + 'host': FLAGS.host}) + self.cloud.allocate_address(self.context) + inst = db.instance_create(self.context, {}) + ec2_id = cloud.internal_id_to_ec2_id(inst['internal_id']) + self.cloud.associate_address(self.context, + instance_id=ec2_id, + public_ip=address) + self.cloud.disassociate_address(self.context, + public_ip=address) + self.cloud.release_address(self.context, + public_ip=address) + db.instance_destroy(self.context, inst['id']) + def test_console_output(self): image_id = FLAGS.default_image instance_type = FLAGS.default_instance_type From 4c002745845fd7954b4af67b6355898aa24b4fca Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 17 Nov 2010 02:23:20 +0000 Subject: [PATCH 36/90] fix leaking floating ip from network unittests and use of fakeldap driver --- nova/tests/network_unittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index b7caed4f..6f470571 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -41,7 +41,6 @@ class NetworkTestCase(test.TrialTestCase): # flags in the corresponding section in nova-dhcpbridge self.flags(connection_type='fake', fake_network=True, - auth_driver='nova.auth.ldapdriver.FakeLdapDriver', network_size=16, num_networks=5) logging.getLogger().setLevel(logging.DEBUG) @@ -127,6 +126,7 @@ class NetworkTestCase(test.TrialTestCase): self.network.deallocate_floating_ip(self.context, float_addr) self.network.deallocate_fixed_ip(self.context, fix_addr) release_ip(fix_addr) + db.floating_ip_destroy(context.get_admin_context(), float_addr) def test_allocate_deallocate_fixed_ip(self): """Makes sure that we can allocate and deallocate a fixed ip""" From e488e74b06fc42aaa509084d646e8ea479662ceb Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 17 Nov 2010 02:41:04 +0000 Subject: [PATCH 37/90] delete floating ips after tests --- nova/tests/cloud_unittest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 938210fa..2c6d9959 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -94,18 +94,19 @@ class CloudTestCase(test.TrialTestCase): def test_describe_addresses(self): """Makes sure describe addresses runs without raising an exception""" address = "10.10.10.10" - db.floating_ip_create(context.get_admin_context(), + db.floating_ip_create(self.context, {'address': address, 'host': FLAGS.host}) self.cloud.allocate_address(self.context) self.cloud.describe_addresses(self.context) self.cloud.release_address(self.context, public_ip=address) + db.floating_ip_destroy(self.context, address) def test_associate_disassociate_address(self): """Verifies associate runs cleanly without raising an exception""" address = "10.10.10.10" - db.floating_ip_create(context.get_admin_context(), + db.floating_ip_create(self.context, {'address': address, 'host': FLAGS.host}) self.cloud.allocate_address(self.context) @@ -119,6 +120,7 @@ class CloudTestCase(test.TrialTestCase): self.cloud.release_address(self.context, public_ip=address) db.instance_destroy(self.context, inst['id']) + db.floating_ip_destroy(self.context, address) def test_console_output(self): image_id = FLAGS.default_image From 4506d3530e3b0ca06ddba62fe2a5ec0bbaf49cc1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 17 Nov 2010 21:23:12 +0000 Subject: [PATCH 38/90] fix greenthread race conditions in trunk and floating ip leakage --- nova/tests/cloud_unittest.py | 4 ++++ nova/tests/quota_unittest.py | 8 +++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 2c6d9959..9886a244 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -101,6 +101,7 @@ class CloudTestCase(test.TrialTestCase): self.cloud.describe_addresses(self.context) self.cloud.release_address(self.context, public_ip=address) + greenthread.sleep(0.3) db.floating_ip_destroy(self.context, address) def test_associate_disassociate_address(self): @@ -111,6 +112,7 @@ class CloudTestCase(test.TrialTestCase): 'host': FLAGS.host}) self.cloud.allocate_address(self.context) inst = db.instance_create(self.context, {}) + fixed = self.network.allocate_fixed_ip(self.context, inst['id']) ec2_id = cloud.internal_id_to_ec2_id(inst['internal_id']) self.cloud.associate_address(self.context, instance_id=ec2_id, @@ -119,6 +121,8 @@ class CloudTestCase(test.TrialTestCase): public_ip=address) self.cloud.release_address(self.context, public_ip=address) + greenthread.sleep(0.3) + self.network.deallocate_fixed_ip(self.context, fixed) db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index 9e3afbf4..b7c1d2ac 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -138,11 +138,8 @@ class QuotaTestCase(test.TrialTestCase): def test_too_many_addresses(self): address = '192.168.0.100' - try: - db.floating_ip_get_by_address(context.get_admin_context(), address) - except exception.NotFound: - db.floating_ip_create(context.get_admin_context(), - {'address': address, 'host': FLAGS.host}) + db.floating_ip_create(context.get_admin_context(), + {'address': address, 'host': FLAGS.host}) float_addr = self.network.allocate_floating_ip(self.context, self.project.id) # NOTE(vish): This assert never fails. When cloud attempts to @@ -151,3 +148,4 @@ class QuotaTestCase(test.TrialTestCase): # that is breaking. self.assertRaises(cloud.QuotaError, self.cloud.allocate_address, self.context) + db.floating_ip_destroy(context.get_admin_context(), address) From 7e466dbd6a840da6401263c46ea72ef0dbb9f1de Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Wed, 17 Nov 2010 16:03:09 -0600 Subject: [PATCH 39/90] really adding images --- .DS_Store | Bin 0 -> 6148 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..d5e4503e74cdef5373b3e177ff8d8c42d3bc2daa GIT binary patch literal 6148 zcmeHK%}T>S5T0$TZvBB=1idYui}cUx!9xfYJPIxJU_}!nG*Ft-r1Vgt5nsqh@F{#A zXLi?OvCx}{&cN>X?aoZH`5?PH03f_+SOahXKw={lL{=D0uXGh`ut3D-Kr#-7ae$-O zU~GNA9cb-E2qA|1VD@$W!zf9s)pt=SmNquGq}-Of{a!Q*m1JK^^?jK%&3a|qI zP665<6gEQFVs21x9oXp-0I`JA+K^K%K{?8zYcV&79yDQ65lyPFC5ABRIInD+YcV%y z(m~kbL)bTe7B51-9iOjsI0)Au_pAUbu&hAQ3>`ZEFYuRHS>!LL@Q4*)1^$= Date: Thu, 18 Nov 2010 13:27:52 -0800 Subject: [PATCH 42/90] First step to getting the image APIs consolidated. The EC2 API was using a one-off S3 image service wrapper, but this should be moved into the nova.image space and use the same interface as the others. There are still some mismatches between the various image service implementations, but this patch was getting large and wanted to keep it within a resonable size. --- nova/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index 4ae86d9b..07b469bc 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -232,7 +232,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.service.LocalImageService', +DEFINE_string('image_service', 'nova.image.local.LocalImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), From 13a59fbbaae361f0ab300dd39176c34d1de54185 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Fri, 19 Nov 2010 09:57:49 -0600 Subject: [PATCH 43/90] Removed .DS_Store files everywhere, begone! --- .DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index d5e4503e74cdef5373b3e177ff8d8c42d3bc2daa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5T0$TZvBB=1idYui}cUx!9xfYJPIxJU_}!nG*Ft-r1Vgt5nsqh@F{#A zXLi?OvCx}{&cN>X?aoZH`5?PH03f_+SOahXKw={lL{=D0uXGh`ut3D-Kr#-7ae$-O zU~GNA9cb-E2qA|1VD@$W!zf9s)pt=SmNquGq}-Of{a!Q*m1JK^^?jK%&3a|qI zP665<6gEQFVs21x9oXp-0I`JA+K^K%K{?8zYcV&79yDQ65lyPFC5ABRIInD+YcV%y z(m~kbL)bTe7B51-9iOjsI0)Au_pAUbu&hAQ3>`ZEFYuRHS>!LL@Q4*)1^$= Date: Fri, 19 Nov 2010 15:47:24 -0600 Subject: [PATCH 44/90] Check for running AMQP instances --- nova/rpc.py | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index ea36d69f..0c79c7f7 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -24,6 +24,7 @@ No fan-out support yet. import json import logging import sys +import time import uuid from carrot import connection as carrot_connection @@ -82,31 +83,35 @@ class Consumer(messaging.Consumer): Contains methods for connecting the fetch method to async loops """ def __init__(self, *args, **kwargs): - self.failed_connection = False - super(Consumer, self).__init__(*args, **kwargs) + while True: + try: + super(Consumer, self).__init__(*args, **kwargs) + break + except: + logging.warning("AMQP server on %s:%d is unreachable. " \ + "Trying again in 30 seconds." % ( + FLAGS.rabbit_host, + FLAGS.rabbit_port + )) + time.sleep(30) + continue def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): """Wraps the parent fetch with some logic for failed connections""" - # TODO(vish): the logic for failed connections and logging should be - # refactored into some sort of connection manager object try: - if self.failed_connection: - # NOTE(vish): conn is defined in the parent class, we can - # recreate it as long as we create the backend too - # pylint: disable-msg=W0201 - self.conn = Connection.recreate() - self.backend = self.conn.create_backend() super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) - if self.failed_connection: - logging.error("Reconnected to queue") - self.failed_connection = False - # NOTE(vish): This is catching all errors because we really don't - # exceptions to be logged 10 times a second if some - # persistent failure occurs. - except Exception: # pylint: disable-msg=W0703 - if not self.failed_connection: - logging.exception("Failed to fetch message from queue") - self.failed_connection = True + except: + try: + self.connection = Connection.recreate() + self.backend = self.connection.create_backend() + self.declare() + except: + logging.warning("AMQP server on %s:%d is unreachable. " \ + "Trying again in 30 seconds." % ( + FLAGS.rabbit_host, + FLAGS.rabbit_port + )) + time.sleep(30) def attach_to_eventlet(self): """Only needed for unit tests!""" From f5af4acdfd438ecddef50714c32c2253cba8ca95 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 19 Nov 2010 16:01:55 -0600 Subject: [PATCH 45/90] Added some comments --- nova/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 0c79c7f7..d9097afa 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -100,12 +100,12 @@ class Consumer(messaging.Consumer): """Wraps the parent fetch with some logic for failed connections""" try: super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) - except: + except: # Catching all because carrot sucks try: self.connection = Connection.recreate() self.backend = self.connection.create_backend() self.declare() - except: + except: # Catching all because carrot sucks logging.warning("AMQP server on %s:%d is unreachable. " \ "Trying again in 30 seconds." % ( FLAGS.rabbit_host, From 3bb2b2b4be992d613b8f06e3d36621a8044417f4 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 19 Nov 2010 16:13:04 -0600 Subject: [PATCH 46/90] Added some comments --- nova/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index d9097afa..cf3889a4 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -87,7 +87,7 @@ class Consumer(messaging.Consumer): try: super(Consumer, self).__init__(*args, **kwargs) break - except: + except: # Catching all because carrot sucks logging.warning("AMQP server on %s:%d is unreachable. " \ "Trying again in 30 seconds." % ( FLAGS.rabbit_host, From bc4a038e3bdce75de57f616abe9b938de55beab6 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 19 Nov 2010 16:42:37 -0600 Subject: [PATCH 47/90] Reverted some changes --- nova/rpc.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index cf3889a4..c7ff9ce6 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -83,35 +83,43 @@ class Consumer(messaging.Consumer): Contains methods for connecting the fetch method to async loops """ def __init__(self, *args, **kwargs): + self.failed_connection = False + while True: try: super(Consumer, self).__init__(*args, **kwargs) break - except: # Catching all because carrot sucks + except: # Catching all because carrot sucks logging.warning("AMQP server on %s:%d is unreachable. " \ "Trying again in 30 seconds." % ( - FLAGS.rabbit_host, - FLAGS.rabbit_port - )) + FLAGS.rabbit_host, + FLAGS.rabbit_port)) time.sleep(30) continue def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): """Wraps the parent fetch with some logic for failed connections""" + # TODO(vish): the logic for failed connections and logging should be + # refactored into some sort of connection manager object try: - super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) - except: # Catching all because carrot sucks - try: + if self.failed_connection: + # NOTE(vish): conn is defined in the parent class, we can + # recreate it as long as we create the backend too + # pylint: disable-msg=W0201 self.connection = Connection.recreate() self.backend = self.connection.create_backend() self.declare() - except: # Catching all because carrot sucks - logging.warning("AMQP server on %s:%d is unreachable. " \ - "Trying again in 30 seconds." % ( - FLAGS.rabbit_host, - FLAGS.rabbit_port - )) - time.sleep(30) + super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) + if self.failed_connection: + logging.error("Reconnected to queue") + self.failed_connection = False + # NOTE(vish): This is catching all errors because we really don't + # exceptions to be logged 10 times a second if some + # persistent failure occurs. + except Exception: # pylint: disable-msg=W0703 + if not self.failed_connection: + logging.exception("Failed to fetch message from queue") + self.failed_connection = True def attach_to_eventlet(self): """Only needed for unit tests!""" From e92ec422956e1efbd7937702077e2607de2e6fa7 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Sat, 20 Nov 2010 12:16:54 -0600 Subject: [PATCH 48/90] Use logging.exception instead --- nova/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index c7ff9ce6..9938b083 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -90,7 +90,7 @@ class Consumer(messaging.Consumer): super(Consumer, self).__init__(*args, **kwargs) break except: # Catching all because carrot sucks - logging.warning("AMQP server on %s:%d is unreachable. " \ + logging.exception("AMQP server on %s:%d is unreachable. " \ "Trying again in 30 seconds." % ( FLAGS.rabbit_host, FLAGS.rabbit_port)) From 60f4f5b75cfd99c1350d65a14f1bf30229ec62db Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 20 Nov 2010 22:26:15 +0100 Subject: [PATCH 49/90] Make sure all templates are included (at least rescue tempaltes didn't used to be included). --- MANIFEST.in | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4fe5f0b3..982b727a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,9 +13,7 @@ include nova/cloudpipe/client.ovpn.template include nova/compute/fakevirtinstance.xml include nova/compute/interfaces.template include nova/virt/interfaces.template -include nova/virt/libvirt.qemu.xml.template -include nova/virt/libvirt.uml.xml.template -include nova/virt/libvirt.xen.xml.template +include nova/virt/libvirt.*.xml.template include nova/tests/CA/ include nova/tests/CA/cacert.pem include nova/tests/CA/private/ From 242db7aebd2281ab5c2277efb880ae3ec1e30ab2 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 14:45:05 -0600 Subject: [PATCH 50/90] Set and use AMQP retry interval and max retry constants --- nova/rpc.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 9938b083..c2d18cb6 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -38,8 +38,11 @@ from nova import fakerabbit from nova import flags from nova import context + FLAGS = flags.FLAGS +AMQP_RETRY_INT = 10 +AMQP_MAX_RETRIES = 12 LOG = logging.getLogger('amqplib') LOG.setLevel(logging.DEBUG) @@ -85,17 +88,23 @@ class Consumer(messaging.Consumer): def __init__(self, *args, **kwargs): self.failed_connection = False - while True: + for i in range(AMQP_MAX_RETRIES): try: super(Consumer, self).__init__(*args, **kwargs) break except: # Catching all because carrot sucks - logging.exception("AMQP server on %s:%d is unreachable. " \ - "Trying again in 30 seconds." % ( - FLAGS.rabbit_host, - FLAGS.rabbit_port)) - time.sleep(30) - continue + if i + 1 == AMQP_MAX_RETRIES: + logging.exception("Unable to connect to AMQP server" \ + " after %d tries. Shutting down." % AMQP_MAX_RETRIES) + sys.exit(1) + else: + logging.exception("AMQP server on %s:%d is unreachable." \ + " Trying again in %d seconds." % ( + FLAGS.rabbit_host, + FLAGS.rabbit_port, + AMQP_RETRY_INT)) + time.sleep(AMQP_RETRY_INT) + continue def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): """Wraps the parent fetch with some logic for failed connections""" @@ -103,7 +112,7 @@ class Consumer(messaging.Consumer): # refactored into some sort of connection manager object try: if self.failed_connection: - # NOTE(vish): conn is defined in the parent class, we can + # NOTE(vish): connection is defined in the parent class, we can # recreate it as long as we create the backend too # pylint: disable-msg=W0201 self.connection = Connection.recreate() From e5b513fe3dca9a1ae3044daf56d7e6cc80804c5e Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 15:22:02 -0600 Subject: [PATCH 51/90] Refactor AMQP retry loop --- nova/rpc.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index c2d18cb6..961b56de 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -86,25 +86,25 @@ class Consumer(messaging.Consumer): Contains methods for connecting the fetch method to async loops """ def __init__(self, *args, **kwargs): - self.failed_connection = False - - for i in range(AMQP_MAX_RETRIES): + for i in xrange(AMQP_MAX_RETRIES): + if i > 0: + time.sleep(AMQP_RETRY_INT) try: super(Consumer, self).__init__(*args, **kwargs) + self.failed_connection = False break except: # Catching all because carrot sucks - if i + 1 == AMQP_MAX_RETRIES: - logging.exception("Unable to connect to AMQP server" \ - " after %d tries. Shutting down." % AMQP_MAX_RETRIES) - sys.exit(1) - else: - logging.exception("AMQP server on %s:%d is unreachable." \ - " Trying again in %d seconds." % ( - FLAGS.rabbit_host, - FLAGS.rabbit_port, - AMQP_RETRY_INT)) - time.sleep(AMQP_RETRY_INT) - continue + logging.exception("AMQP server on %s:%d is unreachable." \ + " Trying again in %d seconds." % ( + FLAGS.rabbit_host, + FLAGS.rabbit_port, + AMQP_RETRY_INT)) + self.failed_connection = True + continue + if self.failed_connection: + logging.exception("Unable to connect to AMQP server" \ + " after %d tries. Shutting down." % AMQP_MAX_RETRIES) + sys.exit(1) def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): """Wraps the parent fetch with some logic for failed connections""" From 770f0e0ab20a073527304f7122d49e936f41c4bc Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 16:04:21 -0600 Subject: [PATCH 52/90] Removed unnecessary continue --- nova/rpc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index 961b56de..57e522ad 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -100,7 +100,6 @@ class Consumer(messaging.Consumer): FLAGS.rabbit_port, AMQP_RETRY_INT)) self.failed_connection = True - continue if self.failed_connection: logging.exception("Unable to connect to AMQP server" \ " after %d tries. Shutting down." % AMQP_MAX_RETRIES) From 4699d65d39aed5020206a6a3feb2de215ae47ed9 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 16:48:44 -0600 Subject: [PATCH 53/90] Use FLAGS instead of constants --- nova/flags.py | 2 ++ nova/rpc.py | 11 ++++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 4ae86d9b..a39f2227 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -196,6 +196,8 @@ DEFINE_integer('rabbit_port', 5672, 'rabbit port') DEFINE_string('rabbit_userid', 'guest', 'rabbit userid') DEFINE_string('rabbit_password', 'guest', 'rabbit password') DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') +DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval') +DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') DEFINE_string('cc_host', '127.0.0.1', 'ip of api server') DEFINE_integer('cc_port', 8773, 'cloud controller port') diff --git a/nova/rpc.py b/nova/rpc.py index 57e522ad..86a29574 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -41,9 +41,6 @@ from nova import context FLAGS = flags.FLAGS -AMQP_RETRY_INT = 10 -AMQP_MAX_RETRIES = 12 - LOG = logging.getLogger('amqplib') LOG.setLevel(logging.DEBUG) @@ -86,9 +83,9 @@ class Consumer(messaging.Consumer): Contains methods for connecting the fetch method to async loops """ def __init__(self, *args, **kwargs): - for i in xrange(AMQP_MAX_RETRIES): + for i in xrange(FLAGS.rabbit_max_retries): if i > 0: - time.sleep(AMQP_RETRY_INT) + time.sleep(FLAGS.rabbit_retry_interval) try: super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False @@ -98,11 +95,11 @@ class Consumer(messaging.Consumer): " Trying again in %d seconds." % ( FLAGS.rabbit_host, FLAGS.rabbit_port, - AMQP_RETRY_INT)) + FLAGS.rabbit_retry_interval)) self.failed_connection = True if self.failed_connection: logging.exception("Unable to connect to AMQP server" \ - " after %d tries. Shutting down." % AMQP_MAX_RETRIES) + " after %d tries. Shutting down." % FLAGS.rabbit_max_retries) sys.exit(1) def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): From e90d5a8e20998f17e3ed43d0c23ba3b06a038a76 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 23 Nov 2010 11:45:56 +0100 Subject: [PATCH 54/90] Unify the location of the default flagfile. Not all workers called utils.default_flagfile, and nova-manage explicitly said to use the one in /etc/nova/nova-manage.conf. This made development awkward since everything but nova-manage would use defaults for everything, but nova-manage would use whatever config was in /etc/nova/nova-manage.conf which was likely put there by a package of some sort. --- bin/nova-compute | 2 ++ bin/nova-instancemonitor | 2 ++ bin/nova-manage | 2 +- bin/nova-network | 2 ++ bin/nova-scheduler | 2 ++ bin/nova-volume | 2 ++ 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bin/nova-compute b/bin/nova-compute index 1724e965..a66477af 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -34,10 +34,12 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from nova import service from nova import twistd +from nova import utils if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': + utils.default_flagfile() application = service.Service.create() # pylint: disable=C0103 diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index 094da403..a7b7fb0c 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -34,6 +34,7 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +from nova import utils from nova import twistd from nova.compute import monitor @@ -44,6 +45,7 @@ if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': + utils.default_flagfile() logging.warn('Starting instance monitor') # pylint: disable-msg=C0103 monitor = monitor.InstanceMonitor() diff --git a/bin/nova-manage b/bin/nova-manage index 08b3da12..eb7c6b87 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -467,7 +467,7 @@ def methods_of(obj): def main(): """Parse options and call the appropriate class/method.""" - utils.default_flagfile('/etc/nova/nova-manage.conf') + utils.default_flagfile() argv = FLAGS(sys.argv) if FLAGS.verbose: diff --git a/bin/nova-network b/bin/nova-network index fa88aeb4..342a6305 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -34,10 +34,12 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from nova import service from nova import twistd +from nova import utils if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': + utils.default_flagfile() application = service.Service.create() # pylint: disable-msg=C0103 diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 38a8f213..069b5a6f 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -34,10 +34,12 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from nova import service from nova import twistd +from nova import utils if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': + utils.default_flagfile() application = service.Service.create() diff --git a/bin/nova-volume b/bin/nova-volume index b9e23571..26148b0e 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -34,10 +34,12 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from nova import service from nova import twistd +from nova import utils if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': + utils.default_flagfile() application = service.Service.create() # pylint: disable-msg=C0103 From afefc42a871d4c56424eecfaf7c0fc6265c98ce4 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 23 Nov 2010 13:48:57 +0100 Subject: [PATCH 55/90] Add a --logdir flag that will be prepended to the logfile setting. This makes it easier to share a flagfile between multiple workers while still having separate log files. --- nova/server.py | 3 +++ nova/twistd.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nova/server.py b/nova/server.py index cb424caa..4d0f6e4d 100644 --- a/nova/server.py +++ b/nova/server.py @@ -42,6 +42,7 @@ flags.DEFINE_bool('daemonize', False, 'daemonize this process') # clutter. flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing') flags.DEFINE_string('logfile', None, 'log file to output to') +flags.DEFINE_string('logdir', None, 'directory to keep log files in (will be prepended to $logfile)') flags.DEFINE_string('pidfile', None, 'pid file to output to') flags.DEFINE_string('working_directory', './', 'working directory...') flags.DEFINE_integer('uid', os.getuid(), 'uid under which to run') @@ -119,6 +120,8 @@ def daemonize(args, name, main): else: if not FLAGS.logfile: FLAGS.logfile = '%s.log' % name + if FLAGS.logdir: + FLAGS.logfile = os.path.join(FLAGS.logdir, FLAGS.logfile) logfile = logging.FileHandler(FLAGS.logfile) logfile.setFormatter(formatter) logger.addHandler(logfile) diff --git a/nova/twistd.py b/nova/twistd.py index 3ec0ff61..6c6cb955 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -43,7 +43,7 @@ else: FLAGS = flags.FLAGS - +flags.DEFINE_string('logdir', None, 'directory to keep log files in (will be prepended to $logfile)') class TwistdServerOptions(ServerOptions): def parseArgs(self, *args): @@ -246,6 +246,8 @@ def serve(filename): FLAGS.logfile = '%s.log' % name elif FLAGS.logfile.endswith('twistd.log'): FLAGS.logfile = FLAGS.logfile.replace('twistd.log', '%s.log' % name) + if FLAGS.logdir: + FLAGS.logfile = os.path.join(FLAGS.logdir, FLAGS.logfile) if not FLAGS.prefix: FLAGS.prefix = name elif FLAGS.prefix.endswith('twisted'): From 1f345d403bcbeb82e88c51bffd393337aef0ddfb Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Tue, 23 Nov 2010 12:04:34 -0600 Subject: [PATCH 56/90] Make aws_access_key_id and aws_secret_access_key configurable --- nova/adminclient.py | 16 +++++++++++----- nova/flags.py | 2 ++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index af55197f..5a62cce7 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -22,13 +22,15 @@ Nova User API client library. import base64 import boto import httplib + +from nova import flags from boto.ec2.regioninfo import RegionInfo +FLAGS = flags.FLAGS + DEFAULT_CLC_URL = 'http://127.0.0.1:8773' DEFAULT_REGION = 'nova' -DEFAULT_ACCESS_KEY = 'admin' -DEFAULT_SECRET_KEY = 'admin' class UserInfo(object): @@ -192,9 +194,13 @@ class HostInfo(object): class NovaAdminClient(object): - def __init__(self, clc_url=DEFAULT_CLC_URL, region=DEFAULT_REGION, - access_key=DEFAULT_ACCESS_KEY, secret_key=DEFAULT_SECRET_KEY, - **kwargs): + def __init__( + self, + clc_url=DEFAULT_CLC_URL, + region=DEFAULT_REGION, + access_key=FLAGS.aws_access_key_id, + secret_key=FLAGS.aws_secret_access_key, + **kwargs): parts = self.split_clc_url(clc_url) self.clc_url = clc_url diff --git a/nova/flags.py b/nova/flags.py index 121b9ca2..f7ae2605 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -179,6 +179,8 @@ DEFINE_list('region_list', [], 'list of region=url pairs separated by commas') DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') +DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID') +DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') From 04f476f167a9d95698bcd407aba46eb654d8f1fb Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 23 Nov 2010 21:52:00 +0100 Subject: [PATCH 58/90] Address PEP8 complaints. --- nova/server.py | 3 ++- nova/twistd.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/server.py b/nova/server.py index 4d0f6e4d..a0ee5468 100644 --- a/nova/server.py +++ b/nova/server.py @@ -42,7 +42,8 @@ flags.DEFINE_bool('daemonize', False, 'daemonize this process') # clutter. flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing') flags.DEFINE_string('logfile', None, 'log file to output to') -flags.DEFINE_string('logdir', None, 'directory to keep log files in (will be prepended to $logfile)') +flags.DEFINE_string('logdir', None, 'directory to keep log files in ' + '(will be prepended to $logfile)') flags.DEFINE_string('pidfile', None, 'pid file to output to') flags.DEFINE_string('working_directory', './', 'working directory...') flags.DEFINE_integer('uid', os.getuid(), 'uid under which to run') diff --git a/nova/twistd.py b/nova/twistd.py index 6c6cb955..cb5648ce 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -43,7 +43,9 @@ else: FLAGS = flags.FLAGS -flags.DEFINE_string('logdir', None, 'directory to keep log files in (will be prepended to $logfile)') +flags.DEFINE_string('logdir', None, 'directory to keep log files in ' + '(will be prepended to $logfile)') + class TwistdServerOptions(ServerOptions): def parseArgs(self, *args): From a118f9e82356c9630a495db599a67f62d25c0a70 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 23 Nov 2010 22:34:53 +0100 Subject: [PATCH 59/90] Add an alias for Armando. --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index ab490606..2a6eb8d7 100644 --- a/.mailmap +++ b/.mailmap @@ -2,6 +2,7 @@ # + From f12220298ba4dfe7c616a820e7ae1106aae512e1 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 24 Nov 2010 22:10:21 +0000 Subject: [PATCH 61/90] Adding support for modification only of user accounts. --- nova/auth/ldapdriver.py | 110 ++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 26 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index ceade1d6..d1ef37cf 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -40,6 +40,8 @@ flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') +flags.DEFINE_boolean('ldap_user_modify_only', False, + 'Modify attributes for users instead of creating/deleting') flags.DEFINE_string('ldap_project_subtree', 'ou=Groups,dc=example,dc=com', 'OU for Projects') flags.DEFINE_string('role_project_subtree', 'ou=Groups,dc=example,dc=com', @@ -89,8 +91,7 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__find_object(self.__uid_to_dn(uid), - '(objectclass=novaUser)') + attr = self.__get_ldap_user(uid) return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -110,7 +111,12 @@ class LdapDriver(object): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, '(objectclass=novaUser)') - return [self.__to_user(attr) for attr in attrs] + users = [] + for attr in attrs: + user = self.__to_user(attr) + if user != None: + users.append(user) + return users def get_projects(self, uid=None): """Retrieve list of projects""" @@ -125,21 +131,46 @@ class LdapDriver(object): """Create a user""" if self.__user_exists(name): raise exception.Duplicate("LDAP user %s already exists" % name) - attr = [ - ('objectclass', ['person', - 'organizationalPerson', - 'inetOrgPerson', - 'novaUser']), - ('ou', [FLAGS.ldap_user_unit]), - ('uid', [name]), - ('sn', [name]), - ('cn', [name]), - ('secretKey', [secret_key]), - ('accessKey', [access_key]), - ('isAdmin', [str(is_admin).upper()]), - ] - self.conn.add_s(self.__uid_to_dn(name), attr) - return self.__to_user(dict(attr)) + if FLAGS.ldap_user_modify_only: + if self.__ldap_user_exists(name): + # Retrieve user by name + user = self.__get_ldap_user(name) + if user.has_key('accessKey') and user.has_key('secretKey') and user.has_key('isAdmin'): + raise exception.Duplicate("LDAP user %s already exists" % name) + else: + # Entry could be malformed, test for missing attrs. + # Malformed entries are useless, replace attributes found. + attr = [] + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) + else: + attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) + else: + attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_REPLACE, 'isAdmin', [str(is_admin).upper()])) + else: + attr.append((self.ldap.MOD_ADD, 'isAdmin', [str(is_admin).upper()])) + self.conn.modify_s(self.__uid_to_dn(name), attr) + return self.get_user(name) + else: + attr = [ + ('objectclass', ['person', + 'organizationalPerson', + 'inetOrgPerson', + 'novaUser']), + ('ou', [FLAGS.ldap_user_unit]), + ('uid', [name]), + ('sn', [name]), + ('cn', [name]), + ('secretKey', [secret_key]), + ('accessKey', [access_key]), + ('isAdmin', [str(is_admin).upper()]), + ] + self.conn.add_s(self.__uid_to_dn(name), attr) + return self.__to_user(dict(attr)) def create_project(self, name, manager_uid, description=None, member_uids=None): @@ -256,7 +287,21 @@ class LdapDriver(object): if not self.__user_exists(uid): raise exception.NotFound("User %s doesn't exist" % uid) self.__remove_from_all(uid) - self.conn.delete_s(self.__uid_to_dn(uid)) + if FLAGS.ldap_user_modify_only: + # Delete attributes + attr = [] + # Retrieve user by name + user = self.__get_ldap_user(uid) + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_DELETE, 'secretKey', user['secretKey'])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_DELETE, 'accessKey', user['accessKey'])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_DELETE, 'isAdmin', user['isAdmin'])) + self.conn.modify_s(self.__uid_to_dn(uid), attr) + else: + # Delete entry + self.conn.delete_s(self.__uid_to_dn(uid)) def delete_project(self, project_id): """Delete a project""" @@ -265,7 +310,7 @@ class LdapDriver(object): self.__delete_group(project_dn) def modify_user(self, uid, access_key=None, secret_key=None, admin=None): - """Modify an existing project""" + """Modify an existing user""" if not access_key and not secret_key and admin is None: return attr = [] @@ -281,10 +326,20 @@ class LdapDriver(object): """Check if user exists""" return self.get_user(uid) != None + def __ldap_user_exists(self, uid): + """Check if the user exists in ldap""" + return self.__get_ldap_user(uid) != None + def __project_exists(self, project_id): """Check if project exists""" return self.get_project(project_id) != None + def __get_ldap_user(self, uid): + """Retrieve LDAP user entry by id""" + attr = self.__find_object(self.__uid_to_dn(uid), + '(objectclass=novaUser)') + return attr + def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" objects = self.__find_objects(dn, query, scope) @@ -449,12 +504,15 @@ class LdapDriver(object): """Convert ldap attributes to User object""" if attr == None: return None - return { - 'id': attr['uid'][0], - 'name': attr['cn'][0], - 'access': attr['accessKey'][0], - 'secret': attr['secretKey'][0], - 'admin': (attr['isAdmin'][0] == 'TRUE')} + if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): + return { + 'id': attr['uid'][0], + 'name': attr['uid'][0], + 'access': attr['accessKey'][0], + 'secret': attr['secretKey'][0], + 'admin': (attr['isAdmin'][0] == 'TRUE')} + else: + return None def __to_project(self, attr): """Convert ldap attributes to Project object""" From df657cdab5193a0bd3e3d5b8d6aec90579d67551 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 24 Nov 2010 22:34:52 +0000 Subject: [PATCH 62/90] Setting "name" back to "cn", since id and name should be separate --- nova/auth/ldapdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index d1ef37cf..95519d00 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -507,7 +507,7 @@ class LdapDriver(object): if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): return { 'id': attr['uid'][0], - 'name': attr['uid'][0], + 'name': attr['cn'][0], 'access': attr['accessKey'][0], 'secret': attr['secretKey'][0], 'admin': (attr['isAdmin'][0] == 'TRUE')} From f42dbc19af88624ecd6232fd5bfc9efd199b8803 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 24 Nov 2010 14:52:10 -0800 Subject: [PATCH 63/90] Consolidated the start instance logic in the two API classes into a single method. This also cleans up a number of small discrepencies between the two. --- nova/tests/quota_unittest.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index b7c1d2ac..1966b51f 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -94,11 +94,12 @@ class QuotaTestCase(test.TrialTestCase): for i in range(FLAGS.quota_instances): instance_id = self._create_instance() instance_ids.append(instance_id) - self.assertRaises(cloud.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, self.cloud.run_instances, self.context, min_count=1, max_count=1, - instance_type='m1.small') + instance_type='m1.small', + image_id='fake') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -106,11 +107,12 @@ class QuotaTestCase(test.TrialTestCase): instance_ids = [] instance_id = self._create_instance(cores=4) instance_ids.append(instance_id) - self.assertRaises(cloud.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, self.cloud.run_instances, self.context, min_count=1, max_count=1, - instance_type='m1.small') + instance_type='m1.small', + image_id='fake') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -119,7 +121,7 @@ class QuotaTestCase(test.TrialTestCase): for i in range(FLAGS.quota_volumes): volume_id = self._create_volume() volume_ids.append(volume_id) - self.assertRaises(cloud.QuotaError, self.cloud.create_volume, + self.assertRaises(quota.QuotaError, self.cloud.create_volume, self.context, size=10) for volume_id in volume_ids: @@ -129,7 +131,7 @@ class QuotaTestCase(test.TrialTestCase): volume_ids = [] volume_id = self._create_volume(size=20) volume_ids.append(volume_id) - self.assertRaises(cloud.QuotaError, + self.assertRaises(quota.QuotaError, self.cloud.create_volume, self.context, size=10) @@ -146,6 +148,6 @@ class QuotaTestCase(test.TrialTestCase): # make an rpc.call, the test just finishes with OK. It # appears to be something in the magic inline callbacks # that is breaking. - self.assertRaises(cloud.QuotaError, self.cloud.allocate_address, + self.assertRaises(quota.QuotaError, self.cloud.allocate_address, self.context) db.floating_ip_destroy(context.get_admin_context(), address) From dc6d4f3cc1be76115157e374e40c884daf5e5553 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Fri, 26 Nov 2010 17:01:50 +0000 Subject: [PATCH 64/90] This modification should have occured in a different branch. Reverting. --- nova/auth/ldapdriver.py | 110 ++++++++++------------------------------ 1 file changed, 26 insertions(+), 84 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 95519d00..ceade1d6 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -40,8 +40,6 @@ flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') -flags.DEFINE_boolean('ldap_user_modify_only', False, - 'Modify attributes for users instead of creating/deleting') flags.DEFINE_string('ldap_project_subtree', 'ou=Groups,dc=example,dc=com', 'OU for Projects') flags.DEFINE_string('role_project_subtree', 'ou=Groups,dc=example,dc=com', @@ -91,7 +89,8 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__get_ldap_user(uid) + attr = self.__find_object(self.__uid_to_dn(uid), + '(objectclass=novaUser)') return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -111,12 +110,7 @@ class LdapDriver(object): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, '(objectclass=novaUser)') - users = [] - for attr in attrs: - user = self.__to_user(attr) - if user != None: - users.append(user) - return users + return [self.__to_user(attr) for attr in attrs] def get_projects(self, uid=None): """Retrieve list of projects""" @@ -131,46 +125,21 @@ class LdapDriver(object): """Create a user""" if self.__user_exists(name): raise exception.Duplicate("LDAP user %s already exists" % name) - if FLAGS.ldap_user_modify_only: - if self.__ldap_user_exists(name): - # Retrieve user by name - user = self.__get_ldap_user(name) - if user.has_key('accessKey') and user.has_key('secretKey') and user.has_key('isAdmin'): - raise exception.Duplicate("LDAP user %s already exists" % name) - else: - # Entry could be malformed, test for missing attrs. - # Malformed entries are useless, replace attributes found. - attr = [] - if user.has_key('secretKey'): - attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) - else: - attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) - if user.has_key('accessKey'): - attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) - else: - attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) - if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_REPLACE, 'isAdmin', [str(is_admin).upper()])) - else: - attr.append((self.ldap.MOD_ADD, 'isAdmin', [str(is_admin).upper()])) - self.conn.modify_s(self.__uid_to_dn(name), attr) - return self.get_user(name) - else: - attr = [ - ('objectclass', ['person', - 'organizationalPerson', - 'inetOrgPerson', - 'novaUser']), - ('ou', [FLAGS.ldap_user_unit]), - ('uid', [name]), - ('sn', [name]), - ('cn', [name]), - ('secretKey', [secret_key]), - ('accessKey', [access_key]), - ('isAdmin', [str(is_admin).upper()]), - ] - self.conn.add_s(self.__uid_to_dn(name), attr) - return self.__to_user(dict(attr)) + attr = [ + ('objectclass', ['person', + 'organizationalPerson', + 'inetOrgPerson', + 'novaUser']), + ('ou', [FLAGS.ldap_user_unit]), + ('uid', [name]), + ('sn', [name]), + ('cn', [name]), + ('secretKey', [secret_key]), + ('accessKey', [access_key]), + ('isAdmin', [str(is_admin).upper()]), + ] + self.conn.add_s(self.__uid_to_dn(name), attr) + return self.__to_user(dict(attr)) def create_project(self, name, manager_uid, description=None, member_uids=None): @@ -287,21 +256,7 @@ class LdapDriver(object): if not self.__user_exists(uid): raise exception.NotFound("User %s doesn't exist" % uid) self.__remove_from_all(uid) - if FLAGS.ldap_user_modify_only: - # Delete attributes - attr = [] - # Retrieve user by name - user = self.__get_ldap_user(uid) - if user.has_key('secretKey'): - attr.append((self.ldap.MOD_DELETE, 'secretKey', user['secretKey'])) - if user.has_key('accessKey'): - attr.append((self.ldap.MOD_DELETE, 'accessKey', user['accessKey'])) - if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_DELETE, 'isAdmin', user['isAdmin'])) - self.conn.modify_s(self.__uid_to_dn(uid), attr) - else: - # Delete entry - self.conn.delete_s(self.__uid_to_dn(uid)) + self.conn.delete_s(self.__uid_to_dn(uid)) def delete_project(self, project_id): """Delete a project""" @@ -310,7 +265,7 @@ class LdapDriver(object): self.__delete_group(project_dn) def modify_user(self, uid, access_key=None, secret_key=None, admin=None): - """Modify an existing user""" + """Modify an existing project""" if not access_key and not secret_key and admin is None: return attr = [] @@ -326,20 +281,10 @@ class LdapDriver(object): """Check if user exists""" return self.get_user(uid) != None - def __ldap_user_exists(self, uid): - """Check if the user exists in ldap""" - return self.__get_ldap_user(uid) != None - def __project_exists(self, project_id): """Check if project exists""" return self.get_project(project_id) != None - def __get_ldap_user(self, uid): - """Retrieve LDAP user entry by id""" - attr = self.__find_object(self.__uid_to_dn(uid), - '(objectclass=novaUser)') - return attr - def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" objects = self.__find_objects(dn, query, scope) @@ -504,15 +449,12 @@ class LdapDriver(object): """Convert ldap attributes to User object""" if attr == None: return None - if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): - return { - 'id': attr['uid'][0], - 'name': attr['cn'][0], - 'access': attr['accessKey'][0], - 'secret': attr['secretKey'][0], - 'admin': (attr['isAdmin'][0] == 'TRUE')} - else: - return None + return { + 'id': attr['uid'][0], + 'name': attr['cn'][0], + 'access': attr['accessKey'][0], + 'secret': attr['secretKey'][0], + 'admin': (attr['isAdmin'][0] == 'TRUE')} def __to_project(self, attr): """Convert ldap attributes to Project object""" From e0db44d68bad0569857a190bad14cc094c2cdda6 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Fri, 26 Nov 2010 17:04:27 +0000 Subject: [PATCH 65/90] Adding support for modification only of user accounts. --- nova/auth/ldapdriver.py | 110 ++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 26 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index ceade1d6..95519d00 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -40,6 +40,8 @@ flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') +flags.DEFINE_boolean('ldap_user_modify_only', False, + 'Modify attributes for users instead of creating/deleting') flags.DEFINE_string('ldap_project_subtree', 'ou=Groups,dc=example,dc=com', 'OU for Projects') flags.DEFINE_string('role_project_subtree', 'ou=Groups,dc=example,dc=com', @@ -89,8 +91,7 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__find_object(self.__uid_to_dn(uid), - '(objectclass=novaUser)') + attr = self.__get_ldap_user(uid) return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -110,7 +111,12 @@ class LdapDriver(object): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, '(objectclass=novaUser)') - return [self.__to_user(attr) for attr in attrs] + users = [] + for attr in attrs: + user = self.__to_user(attr) + if user != None: + users.append(user) + return users def get_projects(self, uid=None): """Retrieve list of projects""" @@ -125,21 +131,46 @@ class LdapDriver(object): """Create a user""" if self.__user_exists(name): raise exception.Duplicate("LDAP user %s already exists" % name) - attr = [ - ('objectclass', ['person', - 'organizationalPerson', - 'inetOrgPerson', - 'novaUser']), - ('ou', [FLAGS.ldap_user_unit]), - ('uid', [name]), - ('sn', [name]), - ('cn', [name]), - ('secretKey', [secret_key]), - ('accessKey', [access_key]), - ('isAdmin', [str(is_admin).upper()]), - ] - self.conn.add_s(self.__uid_to_dn(name), attr) - return self.__to_user(dict(attr)) + if FLAGS.ldap_user_modify_only: + if self.__ldap_user_exists(name): + # Retrieve user by name + user = self.__get_ldap_user(name) + if user.has_key('accessKey') and user.has_key('secretKey') and user.has_key('isAdmin'): + raise exception.Duplicate("LDAP user %s already exists" % name) + else: + # Entry could be malformed, test for missing attrs. + # Malformed entries are useless, replace attributes found. + attr = [] + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) + else: + attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) + else: + attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_REPLACE, 'isAdmin', [str(is_admin).upper()])) + else: + attr.append((self.ldap.MOD_ADD, 'isAdmin', [str(is_admin).upper()])) + self.conn.modify_s(self.__uid_to_dn(name), attr) + return self.get_user(name) + else: + attr = [ + ('objectclass', ['person', + 'organizationalPerson', + 'inetOrgPerson', + 'novaUser']), + ('ou', [FLAGS.ldap_user_unit]), + ('uid', [name]), + ('sn', [name]), + ('cn', [name]), + ('secretKey', [secret_key]), + ('accessKey', [access_key]), + ('isAdmin', [str(is_admin).upper()]), + ] + self.conn.add_s(self.__uid_to_dn(name), attr) + return self.__to_user(dict(attr)) def create_project(self, name, manager_uid, description=None, member_uids=None): @@ -256,7 +287,21 @@ class LdapDriver(object): if not self.__user_exists(uid): raise exception.NotFound("User %s doesn't exist" % uid) self.__remove_from_all(uid) - self.conn.delete_s(self.__uid_to_dn(uid)) + if FLAGS.ldap_user_modify_only: + # Delete attributes + attr = [] + # Retrieve user by name + user = self.__get_ldap_user(uid) + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_DELETE, 'secretKey', user['secretKey'])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_DELETE, 'accessKey', user['accessKey'])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_DELETE, 'isAdmin', user['isAdmin'])) + self.conn.modify_s(self.__uid_to_dn(uid), attr) + else: + # Delete entry + self.conn.delete_s(self.__uid_to_dn(uid)) def delete_project(self, project_id): """Delete a project""" @@ -265,7 +310,7 @@ class LdapDriver(object): self.__delete_group(project_dn) def modify_user(self, uid, access_key=None, secret_key=None, admin=None): - """Modify an existing project""" + """Modify an existing user""" if not access_key and not secret_key and admin is None: return attr = [] @@ -281,10 +326,20 @@ class LdapDriver(object): """Check if user exists""" return self.get_user(uid) != None + def __ldap_user_exists(self, uid): + """Check if the user exists in ldap""" + return self.__get_ldap_user(uid) != None + def __project_exists(self, project_id): """Check if project exists""" return self.get_project(project_id) != None + def __get_ldap_user(self, uid): + """Retrieve LDAP user entry by id""" + attr = self.__find_object(self.__uid_to_dn(uid), + '(objectclass=novaUser)') + return attr + def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" objects = self.__find_objects(dn, query, scope) @@ -449,12 +504,15 @@ class LdapDriver(object): """Convert ldap attributes to User object""" if attr == None: return None - return { - 'id': attr['uid'][0], - 'name': attr['cn'][0], - 'access': attr['accessKey'][0], - 'secret': attr['secretKey'][0], - 'admin': (attr['isAdmin'][0] == 'TRUE')} + if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): + return { + 'id': attr['uid'][0], + 'name': attr['cn'][0], + 'access': attr['accessKey'][0], + 'secret': attr['secretKey'][0], + 'admin': (attr['isAdmin'][0] == 'TRUE')} + else: + return None def __to_project(self, attr): """Convert ldap attributes to Project object""" From 0f02838ef0feb2bad56ded77695590fa7fca8f96 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Fri, 26 Nov 2010 17:59:48 +0000 Subject: [PATCH 66/90] PEP fixes --- nova/auth/ldapdriver.py | 72 ++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 95519d00..fa48c843 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -91,7 +91,7 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__get_ldap_user(uid) + attr = self.__get_ldap_user(uid) return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -111,11 +111,11 @@ class LdapDriver(object): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, '(objectclass=novaUser)') - users = [] - for attr in attrs: - user = self.__to_user(attr) - if user != None: - users.append(user) + users = [] + for attr in attrs: + user = self.__to_user(attr) + if user is not None: + users.append(user) return users def get_projects(self, uid=None): @@ -135,24 +135,32 @@ class LdapDriver(object): if self.__ldap_user_exists(name): # Retrieve user by name user = self.__get_ldap_user(name) - if user.has_key('accessKey') and user.has_key('secretKey') and user.has_key('isAdmin'): - raise exception.Duplicate("LDAP user %s already exists" % name) + if user.has_key('accessKey') and user.has_key('secretKey') \ + and user.has_key('isAdmin'): + raise exception.Duplicate("LDAP user %s already exists" \ + % name) else: # Entry could be malformed, test for missing attrs. # Malformed entries are useless, replace attributes found. attr = [] if user.has_key('secretKey'): - attr.append((self.ldap.MOD_REPLACE, 'secretKey', [secret_key])) + attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ + [secret_key])) else: - attr.append((self.ldap.MOD_ADD, 'secretKey', [secret_key])) + attr.append((self.ldap.MOD_ADD, 'secretKey', \ + [secret_key])) if user.has_key('accessKey'): - attr.append((self.ldap.MOD_REPLACE, 'accessKey', [access_key])) + attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ + [access_key])) else: - attr.append((self.ldap.MOD_ADD, 'accessKey', [access_key])) + attr.append((self.ldap.MOD_ADD, 'accessKey', \ + [access_key])) if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_REPLACE, 'isAdmin', [str(is_admin).upper()])) + attr.append((self.ldap.MOD_REPLACE, 'isAdmin', \ + [str(is_admin).upper()])) else: - attr.append((self.ldap.MOD_ADD, 'isAdmin', [str(is_admin).upper()])) + attr.append((self.ldap.MOD_ADD, 'isAdmin', \ + [str(is_admin).upper()])) self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) else: @@ -186,7 +194,7 @@ class LdapDriver(object): if description is None: description = name members = [] - if member_uids != None: + if member_uids is not None: for member_uid in member_uids: if not self.__user_exists(member_uid): raise exception.NotFound("Project can't be created " @@ -293,11 +301,14 @@ class LdapDriver(object): # Retrieve user by name user = self.__get_ldap_user(uid) if user.has_key('secretKey'): - attr.append((self.ldap.MOD_DELETE, 'secretKey', user['secretKey'])) + attr.append((self.ldap.MOD_DELETE, 'secretKey', \ + user['secretKey'])) if user.has_key('accessKey'): - attr.append((self.ldap.MOD_DELETE, 'accessKey', user['accessKey'])) + attr.append((self.ldap.MOD_DELETE, 'accessKey', \ + user['accessKey'])) if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_DELETE, 'isAdmin', user['isAdmin'])) + attr.append((self.ldap.MOD_DELETE, 'isAdmin', \ + user['isAdmin'])) self.conn.modify_s(self.__uid_to_dn(uid), attr) else: # Delete entry @@ -324,18 +335,18 @@ class LdapDriver(object): def __user_exists(self, uid): """Check if user exists""" - return self.get_user(uid) != None + return self.get_user(uid) is not None def __ldap_user_exists(self, uid): """Check if the user exists in ldap""" - return self.__get_ldap_user(uid) != None + return self.__get_ldap_user(uid) is not None def __project_exists(self, project_id): """Check if project exists""" - return self.get_project(project_id) != None + return self.get_project(project_id) is not None def __get_ldap_user(self, uid): - """Retrieve LDAP user entry by id""" + """Retrieve LDAP user entry by id""" attr = self.__find_object(self.__uid_to_dn(uid), '(objectclass=novaUser)') return attr @@ -385,12 +396,12 @@ class LdapDriver(object): def __group_exists(self, dn): """Check if group exists""" - return self.__find_object(dn, '(objectclass=groupOfNames)') != None + return self.__find_object(dn, '(objectclass=groupOfNames)') is not None @staticmethod def __role_to_dn(role, project_id=None): """Convert role to corresponding dn""" - if project_id == None: + if project_id is None: return FLAGS.__getitem__("ldap_%s" % role).value else: return 'cn=%s,cn=%s,%s' % (role, @@ -404,7 +415,7 @@ class LdapDriver(object): raise exception.Duplicate("Group can't be created because " "group %s already exists" % name) members = [] - if member_uids != None: + if member_uids is not None: for member_uid in member_uids: if not self.__user_exists(member_uid): raise exception.NotFound("Group can't be created " @@ -430,7 +441,7 @@ class LdapDriver(object): res = self.__find_object(group_dn, '(member=%s)' % self.__uid_to_dn(uid), self.ldap.SCOPE_BASE) - return res != None + return res is not None def __add_to_group(self, uid, group_dn): """Add user to group""" @@ -502,21 +513,22 @@ class LdapDriver(object): @staticmethod def __to_user(attr): """Convert ldap attributes to User object""" - if attr == None: + if attr is None: return None - if (attr.has_key('accessKey') and attr.has_key('secretKey') and attr.has_key('isAdmin')): + if (attr.has_key('accessKey') and attr.has_key('secretKey') \ + and attr.has_key('isAdmin')): return { 'id': attr['uid'][0], 'name': attr['cn'][0], 'access': attr['accessKey'][0], 'secret': attr['secretKey'][0], 'admin': (attr['isAdmin'][0] == 'TRUE')} - else: + else: return None def __to_project(self, attr): """Convert ldap attributes to Project object""" - if attr == None: + if attr is None: return None member_dns = attr.get('member', []) return { From 6db7f416504bf2c784cad07eaec9550f98a9b995 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 29 Nov 2010 15:49:12 +0100 Subject: [PATCH 67/90] Make sure templated flags work across calls to ParseNewFlags. ParseNewFlags creates a new FlagValues object, which doesn't have all the previously defined flags, so template lookups fail miserably. Pass the existing FlagValues object too the template mapping object to fix this. --- nova/flags.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 70a04949..641bda5f 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -40,11 +40,12 @@ class FlagValues(gflags.FlagValues): """ - def __init__(self): + def __init__(self, extra_context=None): gflags.FlagValues.__init__(self) self.__dict__['__dirty'] = [] self.__dict__['__was_already_parsed'] = False self.__dict__['__stored_argv'] = [] + self.__dict__['__extra_context'] = extra_context def __call__(self, argv): # We're doing some hacky stuff here so that we don't have to copy @@ -114,7 +115,7 @@ class FlagValues(gflags.FlagValues): def ParseNewFlags(self): if '__stored_argv' not in self.__dict__: return - new_flags = FlagValues() + new_flags = FlagValues(self) for k in self.__dict__['__dirty']: new_flags[k] = gflags.FlagValues.__getitem__(self, k) @@ -139,18 +140,25 @@ class FlagValues(gflags.FlagValues): val = gflags.FlagValues.__getattr__(self, name) if type(val) is str: tmpl = Template(val) - return tmpl.substitute(StrWrapper(self)) + context = [self, self.__dict__['__extra_context']] + return tmpl.substitute(StrWrapper(context)) return val + class StrWrapper(object): - def __init__(self, obj): - self.wrapped = obj + """Wrapper around FlagValues objects + + Wraps FlagValues objects for string.Template so that we're + sure to return strings.""" + def __init__(self, context_objs): + self.context_objs = context_objs def __getitem__(self, name): - if hasattr(self.wrapped, name): - return str(getattr(self.wrapped, name)) - else: - raise KeyError(name) + for context in self.context_objs: + val = getattr(context, name, False) + if val: + return str(val) + raise KeyError(name) FLAGS = FlagValues() gflags.FLAGS = FLAGS From af8e86274a9c26dc5dc67d21fc7a7bed38f4da8a Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 29 Nov 2010 21:05:40 +0100 Subject: [PATCH 68/90] Import string instead of importing Template from string. This is how we do things. --- nova/flags.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 641bda5f..94872944 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -24,10 +24,9 @@ where they're used. import getopt import os import socket +import string import sys -from string import Template - import gflags @@ -139,7 +138,7 @@ class FlagValues(gflags.FlagValues): self.ParseNewFlags() val = gflags.FlagValues.__getattr__(self, name) if type(val) is str: - tmpl = Template(val) + tmpl = string.Template(val) context = [self, self.__dict__['__extra_context']] return tmpl.substitute(StrWrapper(context)) return val From eaadf2f2bf7fbaeca09d1fca255a48c5c77078d7 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 29 Nov 2010 22:01:19 +0100 Subject: [PATCH 69/90] Adjust state_path default setting so that api unit tests find things where they used to find them. --- nova/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index 94872944..cb9fa105 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -244,7 +244,7 @@ DEFINE_string('vpn_key_suffix', DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') -DEFINE_string('state_path', os.path.abspath("./"), +DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), "Top-level directory for maintaining nova's state") DEFINE_string('sql_connection', From 50ecc832f6c3d07327743c9f419db81c36318070 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 30 Nov 2010 23:12:19 +0000 Subject: [PATCH 71/90] Added a script to use OpenDJ as an LDAP server instead of OpenLDAP. Also modified nova.sh to add an USE_OPENDJ option, that will be checked when USE_LDAP is set. --- nova/auth/opendj.sh | 119 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100755 nova/auth/opendj.sh diff --git a/nova/auth/opendj.sh b/nova/auth/opendj.sh new file mode 100755 index 00000000..8052c077 --- /dev/null +++ b/nova/auth/opendj.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# LDAP INSTALL SCRIPT - IS IDEMPOTENT, does not scrub users + +apt-get install -y ldap-utils python-ldap openjdk-6-jre + +if [ ! -d "/usr/opendj" ] +then + # TODO(rlane): Wikimedia Foundation is the current package maintainer. + # After the package is included in Ubuntu's channel, change this. + wget http://apt.wikimedia.org/wikimedia/pool/main/o/opendj/opendj_2.4.0-7_amd64.deb + dpkg -i opendj_2.4.0-7_amd64.deb +fi + +abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` +schemapath='/var/opendj/instance/config/schema' +cp $abspath/openssh-lpk_sun.schema $schemapath/97-openssh-lpk_sun.ldif +cp $abspath/nova_sun.schema $schemapath/98-nova_sun.ldif +chown opendj:opendj $schemapath/97-openssh-lpk_sun.ldif +chown opendj:opendj $schemapath/98-nova_sun.ldif + +cat >/etc/ldap/ldap.conf </etc/ldap/base.ldif < Date: Wed, 1 Dec 2010 11:50:25 +0100 Subject: [PATCH 72/90] Move cc_host and cc_port flags into nova/network/linux_net.py. They weren't used anywhere else. Make cc_host default to nova.utils.get_my_ip() instead of 127.0.0.1. cc_host is used to set up forwarding to the meta-data service, and the kernel doesn't allow routing to a loopback device, so 127.0.0.1 is a poor default. --- nova/flags.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index cb9fa105..1f94feb0 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -223,8 +223,6 @@ DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval') DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') -DEFINE_string('cc_host', '127.0.0.1', 'ip of api server') -DEFINE_integer('cc_port', 8773, 'cloud controller port') DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud', 'Url to ec2 api server') From 67ce9dad05bee1c2c2ce7a1de94c492368d39a43 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Dec 2010 10:44:51 -0600 Subject: [PATCH 73/90] Move default_flagfile() call to where it will be parsed in time to load the flagfile --- bin/nova-compute | 3 ++- bin/nova-instancemonitor | 2 +- bin/nova-network | 2 +- bin/nova-objectstore | 2 +- bin/nova-scheduler | 2 +- bin/nova-volume | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/nova-compute b/bin/nova-compute index a66477af..1054852c 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -38,8 +38,9 @@ from nova import utils if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = service.Service.create() # pylint: disable=C0103 + diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index a7b7fb0c..9b6c40e8 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -42,10 +42,10 @@ logging.getLogger('boto').setLevel(logging.WARN) if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() logging.warn('Starting instance monitor') # pylint: disable-msg=C0103 monitor = monitor.InstanceMonitor() diff --git a/bin/nova-network b/bin/nova-network index 342a6305..d1fb5526 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -38,8 +38,8 @@ from nova import utils if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = service.Service.create() # pylint: disable-msg=C0103 diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 728f2ee5..00ae27af 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -42,8 +42,8 @@ FLAGS = flags.FLAGS if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = handler.get_application() # pylint: disable-msg=C0103 diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 069b5a6f..4d1a40cf 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -38,8 +38,8 @@ from nova import utils if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = service.Service.create() diff --git a/bin/nova-volume b/bin/nova-volume index 26148b0e..e7281d6c 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -38,8 +38,8 @@ from nova import utils if __name__ == '__main__': + utils.default_flagfile() twistd.serve(__file__) if __name__ == '__builtin__': - utils.default_flagfile() application = service.Service.create() # pylint: disable-msg=C0103 From beaa32f0ce57a7cfe278aef7417c8a1d59bad8db Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 1 Dec 2010 09:24:39 -0800 Subject: [PATCH 74/90] Broke parts of compute manager out into compute.api to separate what gets run on the API side vs the worker side. --- nova/tests/compute_unittest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 71a1a445..8f6f35b3 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -31,6 +31,7 @@ from nova import flags from nova import test from nova import utils from nova.auth import manager +from nova.compute import api as compute_api FLAGS = flags.FLAGS @@ -43,6 +44,7 @@ class ComputeTestCase(test.TrialTestCase): self.flags(connection_type='fake', network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) + self.compute_api = compute_api.ComputeAPI() self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') @@ -76,9 +78,9 @@ class ComputeTestCase(test.TrialTestCase): 'user_id': self.user.id, 'project_id': self.project.id} group = db.security_group_create(self.context, values) - ref = self.compute.create_instance(self.context, - security_groups=[group['id']], - **inst) + ref = self.compute_api.create_instance(self.context, + security_groups=[group['id']], + **inst) # reload to get groups instance_ref = db.instance_get(self.context, ref['id']) try: From 4524f81ebfdb5e9f7e46cde9a57c38c1fc0f4444 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 1 Dec 2010 11:43:20 -0600 Subject: [PATCH 75/90] Removed a blank line. --- bin/nova-compute | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/nova-compute b/bin/nova-compute index 1054852c..ac6378f7 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -43,4 +43,3 @@ if __name__ == '__main__': if __name__ == '__builtin__': application = service.Service.create() # pylint: disable=C0103 - From cc9179687565e078059a2acbaf37412b517cfac1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 2 Dec 2010 15:18:45 +0100 Subject: [PATCH 76/90] Add a helpful error message to nova-manage in case of NoMoreNetworks. This is one of the most common problems people have, and the solution is not currently easily discoverable. This should address that. --- bin/nova-manage | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index eb7c6b87..62eec835 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -359,9 +359,14 @@ class ProjectCommands(object): def zipfile(self, project_id, user_id, filename='nova.zip'): """Exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" - zip_file = self.manager.get_credentials(user_id, project_id) - with open(filename, 'w') as f: - f.write(zip_file) + try: + zip_file = self.manager.get_credentials(user_id, project_id) + with open(filename, 'w') as f: + f.write(zip_file) + except db.api.NoMoreNetworks: + print ('No more networks available. If this is a new ' + 'installation, you need\nto call something like this:\n\n' + ' nova-manage network create 10.0.0.0/8 10 64\n\n') class FloatingIpCommands(object): From eabdc25e5b67ca1436bd08697d874314304bf751 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 2 Dec 2010 10:08:56 -0600 Subject: [PATCH 77/90] Default Instance.display_name to a value even when None is explicitly passed in. --- nova/tests/compute_unittest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 71a1a445..85992b48 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -66,6 +66,16 @@ class ComputeTestCase(test.TrialTestCase): inst['ami_launch_index'] = 0 return db.instance_create(self.context, inst)['id'] + def test_create_instance_defaults_display_name(self): + """Verify that an instance cannot be created without a display_name.""" + cases = [dict(), dict(display_name=None)] + for instance in cases: + ref = self.compute.create_instance(self.context, None, **instance) + try: + self.assertNotEqual(ref.display_name, None) + finally: + db.instance_destroy(self.context, ref['id']) + def test_create_instance_associates_security_groups(self): """Make sure create_instance associates security groups""" inst = {} From ad17df5866dab90eb8106b4b0421fc4f72e97f79 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 2 Dec 2010 10:53:32 -0800 Subject: [PATCH 78/90] Added test files to be ignored. --- .bzrignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.bzrignore b/.bzrignore index ab099d3e..82db46fa 100644 --- a/.bzrignore +++ b/.bzrignore @@ -1,3 +1,13 @@ run_tests.err.log .nova-venv ChangeLog +_trial_temp +keys +networks +nova.sqlite +CA/cacert.pem +CA/index.txt* +CA/openssl.cnf +CA/serial* +CA/newcerts/*.pem +CA/private/cakey.pem From fa4810d4a1e41bcf20e4fc27ecbe6493a0c0f44f Mon Sep 17 00:00:00 2001 From: Eric Day Date: Fri, 3 Dec 2010 12:21:18 -0800 Subject: [PATCH 80/90] Finished cleaning up the openstack servers API, it no longer touches the database directly. Also cleaned up similar things in ec2 API and refactored a couple methods in nova.compute.api to accomodate this work. --- nova/auth/manager.py | 4 ++++ nova/flags.py | 2 +- nova/tests/compute_unittest.py | 24 +++++++++--------------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 7b2b6816..11c3bd6d 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -624,6 +624,10 @@ class AuthManager(object): with self.driver() as drv: drv.modify_user(uid, access_key, secret_key, admin) + @staticmethod + def get_key_pairs(context): + return db.key_pair_get_all_by_user(context.elevated(), context.user_id) + def get_credentials(self, user, project=None): """Get credential zip for user in project""" if not isinstance(user, User): diff --git a/nova/flags.py b/nova/flags.py index 1f94feb0..c6578023 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -259,7 +259,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.local.LocalImageService', +DEFINE_string('image_service', 'nova.image.s3.S3ImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index a5544973..6f3ef96c 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -72,33 +72,27 @@ class ComputeTestCase(test.TrialTestCase): """Verify that an instance cannot be created without a display_name.""" cases = [dict(), dict(display_name=None)] for instance in cases: - ref = self.compute_api.create_instance(self.context, None, - **instance) + ref = self.compute_api.create_instances(self.context, + FLAGS.default_instance_type, None, **instance) try: - self.assertNotEqual(ref.display_name, None) + self.assertNotEqual(ref[0].display_name, None) finally: - db.instance_destroy(self.context, ref['id']) + db.instance_destroy(self.context, ref[0]['id']) def test_create_instance_associates_security_groups(self): - """Make sure create_instance associates security groups""" - inst = {} - inst['user_id'] = self.user.id - inst['project_id'] = self.project.id + """Make sure create_instances associates security groups""" values = {'name': 'default', 'description': 'default', 'user_id': self.user.id, 'project_id': self.project.id} group = db.security_group_create(self.context, values) - ref = self.compute_api.create_instance(self.context, - security_groups=[group['id']], - **inst) - # reload to get groups - instance_ref = db.instance_get(self.context, ref['id']) + ref = self.compute_api.create_instances(self.context, + FLAGS.default_instance_type, None, security_group=['default']) try: - self.assertEqual(len(instance_ref['security_groups']), 1) + self.assertEqual(len(ref[0]['security_groups']), 1) finally: db.security_group_destroy(self.context, group['id']) - db.instance_destroy(self.context, instance_ref['id']) + db.instance_destroy(self.context, ref[0]['id']) @defer.inlineCallbacks def test_run_terminate(self): From d537cc4dd369a5e40c7073fcce2d698e7371dc9b Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 7 Dec 2010 20:25:24 +0100 Subject: [PATCH 82/90] Add Ryan Lucio to Authors --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index ef1a535c..62f0c49d 100644 --- a/Authors +++ b/Authors @@ -20,6 +20,7 @@ Michael Gundlach Monty Taylor Paul Voccio Rick Clark +Ryan Lucio Soren Hansen Todd Willey Vishvananda Ishaya From 9805e5d84ed266c2e8bc3cce8e6393d6cf3b64e0 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 7 Dec 2010 19:35:05 +0000 Subject: [PATCH 83/90] Adding myself to the authors list --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index ef1a535c..a35398d5 100644 --- a/Authors +++ b/Authors @@ -25,3 +25,4 @@ Todd Willey Vishvananda Ishaya Youcef Laribi Zhixue Wu +Ryan Lane From fc016e9731a57b98eba8da7a613d1bc7636f9a1f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 7 Dec 2010 21:35:15 +0100 Subject: [PATCH 84/90] Make sure Authors check also works for pending merges (otherwise stuff can get merged that will make the next merge fail this check). --- nova/tests/misc_unittest.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/nova/tests/misc_unittest.py b/nova/tests/misc_unittest.py index 856060af..667c63ad 100644 --- a/nova/tests/misc_unittest.py +++ b/nova/tests/misc_unittest.py @@ -15,7 +15,6 @@ # under the License. import os -import subprocess from nova import test from nova.utils import parse_mailmap, str_dict_replace @@ -24,18 +23,23 @@ from nova.utils import parse_mailmap, str_dict_replace class ProjectTestCase(test.TrialTestCase): def test_authors_up_to_date(self): if os.path.exists('../.bzr'): - log_cmd = subprocess.Popen(["bzr", "log", "-n0"], - stdout=subprocess.PIPE) - changelog = log_cmd.communicate()[0] + contributors = set() + mailmap = parse_mailmap('../.mailmap') - contributors = set() - for l in changelog.split('\n'): - l = l.strip() - if (l.startswith('author:') or l.startswith('committer:') - and not l == 'committer: Tarmac'): - email = l.split(' ')[-1] - contributors.add(str_dict_replace(email, mailmap)) + import bzrlib.workingtree + tree = bzrlib.workingtree.WorkingTree.open('..') + tree.lock_read() + parents = tree.get_parent_ids() + g = tree.branch.repository.get_graph() + for p in parents[1:]: + rev_ids = [r for r, _ in g.iter_ancestry(parents) + if r != "null:"] + revs = tree.branch.repository.get_revisions(rev_ids) + for r in revs: + for author in r.get_apparent_authors(): + email = author.split(' ')[-1] + contributors.add(str_dict_replace(email, mailmap)) authors_file = open('../Authors', 'r').read() From 5732f1ab06e95c887fdc570e8f8fcb0cc1f61bc8 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 7 Dec 2010 21:13:54 +0000 Subject: [PATCH 85/90] Reverting last change --- Authors | 1 - 1 file changed, 1 deletion(-) diff --git a/Authors b/Authors index a35398d5..ef1a535c 100644 --- a/Authors +++ b/Authors @@ -25,4 +25,3 @@ Todd Willey Vishvananda Ishaya Youcef Laribi Zhixue Wu -Ryan Lane From 274c7d19c34e76247cf64f05297e0f03c95aab77 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 7 Dec 2010 23:46:18 +0000 Subject: [PATCH 86/90] Removing redundant check --- nova/auth/ldapdriver.py | 49 ++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index fa48c843..d54a0dfa 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -135,34 +135,29 @@ class LdapDriver(object): if self.__ldap_user_exists(name): # Retrieve user by name user = self.__get_ldap_user(name) - if user.has_key('accessKey') and user.has_key('secretKey') \ - and user.has_key('isAdmin'): - raise exception.Duplicate("LDAP user %s already exists" \ - % name) + # Entry could be malformed, test for missing attrs. + # Malformed entries are useless, replace attributes found. + attr = [] + if user.has_key('secretKey'): + attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ + [secret_key])) else: - # Entry could be malformed, test for missing attrs. - # Malformed entries are useless, replace attributes found. - attr = [] - if user.has_key('secretKey'): - attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ - [secret_key])) - else: - attr.append((self.ldap.MOD_ADD, 'secretKey', \ - [secret_key])) - if user.has_key('accessKey'): - attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ - [access_key])) - else: - attr.append((self.ldap.MOD_ADD, 'accessKey', \ - [access_key])) - if user.has_key('isAdmin'): - attr.append((self.ldap.MOD_REPLACE, 'isAdmin', \ - [str(is_admin).upper()])) - else: - attr.append((self.ldap.MOD_ADD, 'isAdmin', \ - [str(is_admin).upper()])) - self.conn.modify_s(self.__uid_to_dn(name), attr) - return self.get_user(name) + attr.append((self.ldap.MOD_ADD, 'secretKey', \ + [secret_key])) + if user.has_key('accessKey'): + attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ + [access_key])) + else: + attr.append((self.ldap.MOD_ADD, 'accessKey', \ + [access_key])) + if user.has_key('isAdmin'): + attr.append((self.ldap.MOD_REPLACE, 'isAdmin', \ + [str(is_admin).upper()])) + else: + attr.append((self.ldap.MOD_ADD, 'isAdmin', \ + [str(is_admin).upper()])) + self.conn.modify_s(self.__uid_to_dn(name), attr) + return self.get_user(name) else: attr = [ ('objectclass', ['person', From 85f64cff078fa06d1dfeb89b99910f9c935a2145 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Tue, 7 Dec 2010 23:53:01 +0000 Subject: [PATCH 87/90] Raising an exception if the user doesn't exist before trying to modify its attributes --- nova/auth/ldapdriver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index d54a0dfa..5727c8da 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -158,6 +158,8 @@ class LdapDriver(object): [str(is_admin).upper()])) self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) + else: + raise exception.NotFound("User %s doesn't exist" % name) else: attr = [ ('objectclass', ['person', From 601758a2892bea185864a68a627184763ccf6488 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 00:08:47 +0000 Subject: [PATCH 88/90] Clarifying previously commited exception message --- nova/auth/ldapdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 5727c8da..45ea0683 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -159,7 +159,7 @@ class LdapDriver(object): self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) else: - raise exception.NotFound("User %s doesn't exist" % name) + raise exception.NotFound("LDAP object for %s doesn't exist" % name) else: attr = [ ('objectclass', ['person', From dbf1a026c24d8b11ad875fc74f990ae81b77a094 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 00:26:41 +0000 Subject: [PATCH 89/90] pep8 fix --- nova/auth/ldapdriver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 45ea0683..9baf45c9 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -159,7 +159,8 @@ class LdapDriver(object): self.conn.modify_s(self.__uid_to_dn(name), attr) return self.get_user(name) else: - raise exception.NotFound("LDAP object for %s doesn't exist" % name) + raise exception.NotFound("LDAP object for %s doesn't exist" + % name) else: attr = [ ('objectclass', ['person', From 92666b693a1eff2faa653ecf46bf040a3eabb881 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Wed, 8 Dec 2010 00:34:20 +0000 Subject: [PATCH 90/90] More pep8 fixes to remove deprecated functions --- nova/auth/ldapdriver.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 9baf45c9..c10939d7 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -138,19 +138,19 @@ class LdapDriver(object): # Entry could be malformed, test for missing attrs. # Malformed entries are useless, replace attributes found. attr = [] - if user.has_key('secretKey'): + if 'secretKey' in user.keys(): attr.append((self.ldap.MOD_REPLACE, 'secretKey', \ [secret_key])) else: attr.append((self.ldap.MOD_ADD, 'secretKey', \ [secret_key])) - if user.has_key('accessKey'): + if 'accessKey' in user.keys(): attr.append((self.ldap.MOD_REPLACE, 'accessKey', \ [access_key])) else: attr.append((self.ldap.MOD_ADD, 'accessKey', \ [access_key])) - if user.has_key('isAdmin'): + if 'isAdmin' in user.keys(): attr.append((self.ldap.MOD_REPLACE, 'isAdmin', \ [str(is_admin).upper()])) else: @@ -298,13 +298,13 @@ class LdapDriver(object): attr = [] # Retrieve user by name user = self.__get_ldap_user(uid) - if user.has_key('secretKey'): + if 'secretKey' in user.keys(): attr.append((self.ldap.MOD_DELETE, 'secretKey', \ user['secretKey'])) - if user.has_key('accessKey'): + if 'accessKey' in user.keys(): attr.append((self.ldap.MOD_DELETE, 'accessKey', \ user['accessKey'])) - if user.has_key('isAdmin'): + if 'isAdmin' in user.keys(): attr.append((self.ldap.MOD_DELETE, 'isAdmin', \ user['isAdmin'])) self.conn.modify_s(self.__uid_to_dn(uid), attr) @@ -513,8 +513,8 @@ class LdapDriver(object): """Convert ldap attributes to User object""" if attr is None: return None - if (attr.has_key('accessKey') and attr.has_key('secretKey') \ - and attr.has_key('isAdmin')): + if ('accessKey' in attr.keys() and 'secretKey' in attr.keys() \ + and 'isAdmin' in attr.keys()): return { 'id': attr['uid'][0], 'name': attr['cn'][0],