From 3aaad0f0bd26747071406dc5de2e9ecec23e62ae Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 20 Jul 2011 15:38:29 -0500 Subject: [PATCH 01/91] Fix plus passing tests --- nova/tests/test_xenapi.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 4cb7447d3..9b512b73b 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -785,6 +785,45 @@ class XenAPIMigrateInstance(test.TestCase): def test_finish_resize(self): instance = db.instance_create(self.context, self.values) + self.called = False + + def fake_vdi_resize(*args, **kwargs): + self.called = True + + self.stubs.Set(stubs.FakeSessionForMigrationTests, + "VDI_resize_online", fake_vdi_resize) + stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) + stubs.stubout_loopingcall_start(self.stubs) + conn = xenapi_conn.get_connection(False) + network_info = [({'bridge': 'fa0', 'id': 0, 'injected': False}, + {'broadcast': '192.168.0.255', + 'dns': ['192.168.0.1'], + 'gateway': '192.168.0.1', + 'gateway6': 'dead:beef::1', + 'ip6s': [{'enabled': '1', + 'ip': 'dead:beef::dcad:beff:feef:0', + 'netmask': '64'}], + 'ips': [{'enabled': '1', + 'ip': '192.168.0.100', + 'netmask': '255.255.255.0'}], + 'label': 'fake', + 'mac': 'DE:AD:BE:EF:00:00', + 'rxtx_cap': 3})] + conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), + network_info) + self.assertEqual(self.called, True) + + def test_finish_migrate_no_resize_vdi(self): + tiny_type_id = \ + instance_types.get_instance_type_by_name('m1.tiny')['id'] + self.values.update({'instance_type_id': tiny_type_id, 'local_gb': 0}) + instance = db.instance_create(self.context, self.values) + + def fake_vdi_resize(*args, **kwargs): + raise Exception("This shouldn't be called") + + self.stubs.Set(stubs.FakeSessionForMigrationTests, + "VDI_resize_online", fake_vdi_resize) stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) stubs.stubout_loopingcall_start(self.stubs) conn = xenapi_conn.get_connection(False) From 3fdacd1aeaae016c1a147293dd3a36f5599cbf31 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 20 Jul 2011 16:56:45 -0500 Subject: [PATCH 02/91] CHanges based on feedback --- nova/tests/test_xenapi.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 9b512b73b..be263d17c 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -810,10 +810,10 @@ class XenAPIMigrateInstance(test.TestCase): 'mac': 'DE:AD:BE:EF:00:00', 'rxtx_cap': 3})] conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), - network_info) + network_info, resize_instance=True) self.assertEqual(self.called, True) - def test_finish_migrate_no_resize_vdi(self): + def test_finish_migrate_no_local_storage(self): tiny_type_id = \ instance_types.get_instance_type_by_name('m1.tiny')['id'] self.values.update({'instance_type_id': tiny_type_id, 'local_gb': 0}) @@ -842,7 +842,37 @@ class XenAPIMigrateInstance(test.TestCase): 'mac': 'DE:AD:BE:EF:00:00', 'rxtx_cap': 3})] conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), - network_info) + network_info, resize_instance=True) + + def test_finish_migrate_no_resize_vdi(self): + instance = db.instance_create(self.context, self.values) + + def fake_vdi_resize(*args, **kwargs): + raise Exception("This shouldn't be called") + + self.stubs.Set(stubs.FakeSessionForMigrationTests, + "VDI_resize_online", fake_vdi_resize) + stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) + stubs.stubout_loopingcall_start(self.stubs) + conn = xenapi_conn.get_connection(False) + network_info = [({'bridge': 'fa0', 'id': 0, 'injected': False}, + {'broadcast': '192.168.0.255', + 'dns': ['192.168.0.1'], + 'gateway': '192.168.0.1', + 'gateway6': 'dead:beef::1', + 'ip6s': [{'enabled': '1', + 'ip': 'dead:beef::dcad:beff:feef:0', + 'netmask': '64'}], + 'ips': [{'enabled': '1', + 'ip': '192.168.0.100', + 'netmask': '255.255.255.0'}], + 'label': 'fake', + 'mac': 'DE:AD:BE:EF:00:00', + 'rxtx_cap': 3})] + + # Resize instance would be determined by the compute call + conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), + network_info, resize_instance=False) class XenAPIDetermineDiskImageTestCase(test.TestCase): From 35b05fa6e12f5da0d363db36cc8a96fbcdeaf171 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Thu, 21 Jul 2011 12:46:58 -0500 Subject: [PATCH 03/91] Renamed the virt driver resize methods to migration for marginally more understandable code --- nova/tests/test_compute.py | 6 +++--- nova/tests/test_xenapi.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index dc3f0596d..352011e52 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -420,7 +420,7 @@ class ComputeTestCase(test.TestCase): def fake(*args, **kwargs): pass - self.stubs.Set(self.compute.driver, 'finish_resize', fake) + self.stubs.Set(self.compute.driver, 'finish_migration', fake) self.stubs.Set(self.compute.network_api, 'get_instance_nw_info', fake) context = self.context.elevated() instance_id = self._create_instance() @@ -527,8 +527,8 @@ class ComputeTestCase(test.TestCase): def fake(*args, **kwargs): pass - self.stubs.Set(self.compute.driver, 'finish_resize', fake) - self.stubs.Set(self.compute.driver, 'revert_resize', fake) + self.stubs.Set(self.compute.driver, 'finish_migration', fake) + self.stubs.Set(self.compute.driver, 'revert_migration', fake) self.stubs.Set(self.compute.network_api, 'get_instance_nw_info', fake) self.compute.run_instance(self.context, instance_id) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index be263d17c..9f203c477 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -783,7 +783,7 @@ class XenAPIMigrateInstance(test.TestCase): conn = xenapi_conn.get_connection(False) conn.migrate_disk_and_power_off(instance, '127.0.0.1') - def test_finish_resize(self): + def test_finish_migrate(self): instance = db.instance_create(self.context, self.values) self.called = False @@ -809,7 +809,7 @@ class XenAPIMigrateInstance(test.TestCase): 'label': 'fake', 'mac': 'DE:AD:BE:EF:00:00', 'rxtx_cap': 3})] - conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), + conn.finish_migration(instance, dict(base_copy='hurr', cow='durr'), network_info, resize_instance=True) self.assertEqual(self.called, True) @@ -841,7 +841,7 @@ class XenAPIMigrateInstance(test.TestCase): 'label': 'fake', 'mac': 'DE:AD:BE:EF:00:00', 'rxtx_cap': 3})] - conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), + conn.finish_migration(instance, dict(base_copy='hurr', cow='durr'), network_info, resize_instance=True) def test_finish_migrate_no_resize_vdi(self): @@ -871,7 +871,7 @@ class XenAPIMigrateInstance(test.TestCase): 'rxtx_cap': 3})] # Resize instance would be determined by the compute call - conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), + conn.finish_migration(instance, dict(base_copy='hurr', cow='durr'), network_info, resize_instance=False) From ca79d9bdf25901741aa58be523c79145b8ebad2e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 21 Jul 2011 22:46:36 +0000 Subject: [PATCH 04/91] change context to maintain exact time, store roles, use ids instead of objects and use a uuid for request_id --- nova/utils.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 8784a227d..737903f81 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -19,7 +19,6 @@ """Utilities and helper functions.""" -import base64 import datetime import functools import inspect @@ -30,7 +29,6 @@ import os import random import re import socket -import string import struct import sys import time @@ -50,7 +48,8 @@ from nova import version LOG = logging.getLogger("nova.utils") -TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +ISO_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" FLAGS = flags.FLAGS @@ -361,16 +360,26 @@ def clear_time_override(): utcnow.override_time = None -def isotime(at=None): - """Returns iso formatted utcnow.""" +def strtime(at=None, fmt=PERFECT_TIME_FORMAT): + """Returns formatted utcnow.""" if not at: at = utcnow() - return at.strftime(TIME_FORMAT) + return at.strftime(fmt) + + +def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): + """Turn a formatted time back into a datetime.""" + return datetime.datetime.strptime(timestr, fmt) + + +def isotime(at=None): + """Returns iso formatted utcnow.""" + return strtime(at, ISO_TIME_FORMAT) def parse_isotime(timestr): """Turn an iso formatted time back into a datetime.""" - return datetime.datetime.strptime(timestr, TIME_FORMAT) + return parse_strtime(timestr, ISO_TIME_FORMAT) def parse_mailmap(mailmap='.mailmap'): From bb5e9e87a8562c252e8b77cf284616247e761d32 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 21 Jul 2011 22:46:57 +0000 Subject: [PATCH 05/91] start removing references to AuthManager --- nova/log.py | 4 ++-- nova/tests/hyperv_unittest.py | 2 +- nova/tests/scheduler/test_scheduler.py | 17 ++++------------- nova/tests/test_access.py | 2 +- nova/tests/test_adminapi.py | 4 ++-- nova/tests/test_cloud.py | 12 ++++++------ nova/tests/test_libvirt.py | 9 ++++----- 7 files changed, 20 insertions(+), 30 deletions(-) diff --git a/nova/log.py b/nova/log.py index f8c0ba68d..b4f6c1d2e 100644 --- a/nova/log.py +++ b/nova/log.py @@ -43,8 +43,8 @@ from nova import version FLAGS = flags.FLAGS flags.DEFINE_string('logging_context_format_string', '%(asctime)s %(levelname)s %(name)s ' - '[%(request_id)s %(user)s ' - '%(project)s] %(message)s', + '[%(request_id)s %(user_id)s ' + '%(project_id)s] %(message)s', 'format string to use for log messages with context') flags.DEFINE_string('logging_default_format_string', '%(asctime)s %(levelname)s %(name)s [-] ' diff --git a/nova/tests/hyperv_unittest.py b/nova/tests/hyperv_unittest.py index 042819b9c..ab2995923 100644 --- a/nova/tests/hyperv_unittest.py +++ b/nova/tests/hyperv_unittest.py @@ -38,7 +38,7 @@ class HyperVTestCase(test.TestCase): self.user = self.manager.create_user('fake', 'fake', 'fake', admin=True) self.project = self.manager.create_project('fake', 'fake', 'fake') - self.context = context.RequestContext(self.user, self.project) + self.context = context.RequestContext(self.user.id, self.project.id) def test_create_destroy(self): """Create a VM and destroy it""" diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py index daea826fd..ef4ef156c 100644 --- a/nova/tests/scheduler/test_scheduler.py +++ b/nova/tests/scheduler/test_scheduler.py @@ -23,7 +23,6 @@ import datetime import mox import novaclient.exceptions import stubout -import webob from mox import IgnoreArg from nova import context @@ -34,12 +33,10 @@ from nova import service from nova import test from nova import rpc from nova import utils -from nova.auth import manager as auth_manager from nova.scheduler import api from nova.scheduler import manager from nova.scheduler import driver from nova.compute import power_state -from nova.db.sqlalchemy import models FLAGS = flags.FLAGS @@ -250,23 +247,17 @@ class SimpleDriverTestCase(test.TestCase): volume_driver='nova.volume.driver.FakeISCSIDriver', scheduler_driver='nova.scheduler.simple.SimpleScheduler') self.scheduler = manager.SchedulerManager() - self.manager = auth_manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake') - self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.get_admin_context() - - def tearDown(self): - self.manager.delete_user(self.user) - self.manager.delete_project(self.project) - super(SimpleDriverTestCase, self).tearDown() + self.user_id = 'fake' + self.project_id = 'fake' def _create_instance(self, **kwargs): """Create a test instance""" inst = {} inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' - inst['user_id'] = self.user.id - inst['project_id'] = self.project.id + inst['user_id'] = self.user_id + inst['project_id'] = self.project_id inst['instance_type_id'] = '1' inst['vcpus'] = kwargs.get('vcpus', 1) inst['ami_launch_index'] = 0 diff --git a/nova/tests/test_access.py b/nova/tests/test_access.py index e170ccee6..6069c5d71 100644 --- a/nova/tests/test_access.py +++ b/nova/tests/test_access.py @@ -93,7 +93,7 @@ class AccessTestCase(test.TestCase): super(AccessTestCase, self).tearDown() def response_status(self, user, methodName): - ctxt = context.RequestContext(user, self.project) + ctxt = context.RequestContext(user.id, self.project.id) environ = self._env_for(ctxt, methodName) req = webob.Request.blank('/', environ) resp = req.get_response(self.mw) diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py index 877cf4ea1..f8abe609d 100644 --- a/nova/tests/test_adminapi.py +++ b/nova/tests/test_adminapi.py @@ -54,8 +54,8 @@ class AdminApiTestCase(test.TestCase): self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('proj', 'admin', 'proj') - self.context = context.RequestContext(user=self.user, - project=self.project) + self.context = context.RequestContext(user_id=self.user.id, + project_id=self.project.id) def fake_show(meh, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 8cdc73a66..71ac7f473 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -65,8 +65,8 @@ class CloudTestCase(test.TestCase): self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('proj', 'admin', 'proj') - self.context = context.RequestContext(user=self.user, - project=self.project) + self.context = context.RequestContext(user_id=self.user.id, + project_id=self.project.id) host = self.network.host def fake_show(meh, context, id): @@ -97,7 +97,7 @@ class CloudTestCase(test.TestCase): def _create_key(self, name): # NOTE(vish): create depends on pool, so just call helper directly - return cloud._gen_key(self.context, self.context.user.id, name) + return cloud._gen_key(self.context, self.context.user_id, name) def test_describe_regions(self): """Makes sure describe regions runs without raising an exception""" @@ -936,7 +936,7 @@ class CloudTestCase(test.TestCase): key = RSA.load_key_string(private_key, callback=lambda: None) bio = BIO.MemoryBuffer() public_key = db.key_pair_get(self.context, - self.context.user.id, + self.context.user_id, 'test')['public_key'] key.save_pub_key_bio(bio) converted = crypto.ssl_pub_to_ssh_pub(bio.read()) @@ -960,7 +960,7 @@ class CloudTestCase(test.TestCase): 'mytestfprint') self.assertTrue(result1) keydata = db.key_pair_get(self.context, - self.context.user.id, + self.context.user_id, 'testimportkey1') self.assertEqual('mytestpubkey', keydata['public_key']) self.assertEqual('mytestfprint', keydata['fingerprint']) @@ -977,7 +977,7 @@ class CloudTestCase(test.TestCase): dummypub) self.assertTrue(result2) keydata = db.key_pair_get(self.context, - self.context.user.id, + self.context.user_id, 'testimportkey2') self.assertEqual(dummypub, keydata['public_key']) self.assertEqual(dummyfprint, keydata['fingerprint']) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 6e2ec7ed6..948ca215f 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -429,8 +429,8 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(parameters[1].get('value'), 'fake') def _check_xml_and_container(self, instance): - user_context = context.RequestContext(project=self.project, - user=self.user) + user_context = context.RequestContext(self.user.id, + self.project.id) instance_ref = db.instance_create(user_context, instance) # Re-get the instance so it's bound to an actual session instance_ref = db.instance_get(user_context, instance_ref['id']) @@ -475,8 +475,7 @@ class LibvirtConnTestCase(test.TestCase): def _check_xml_and_uri(self, instance, expect_ramdisk, expect_kernel, rescue=False): - user_context = context.RequestContext(project=self.project, - user=self.user) + user_context = context.RequestContext(self.user.id, self.project.id) instance_ref = db.instance_create(user_context, instance) network_ref = db.project_get_networks(context.get_admin_context(), self.project.id)[0] @@ -1166,7 +1165,7 @@ class NWFilterTestCase(test.TestCase): self.user = self.manager.create_user('fake', 'fake', 'fake', admin=True) self.project = self.manager.create_project('fake', 'fake', 'fake') - self.context = context.RequestContext(self.user, self.project) + self.context = context.RequestContext(self.user.id, self.project.id) self.fake_libvirt_connection = Mock() From d764ee00e463e1e911b27402c32b7b065fc2a211 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 22 Jul 2011 00:39:53 +0000 Subject: [PATCH 06/91] fix a whole bunch of tests --- nova/tests/hyperv_unittest.py | 9 ++-- nova/tests/test_adminapi.py | 16 +++---- nova/tests/test_api.py | 74 +++--------------------------- nova/tests/test_cloud.py | 16 +++---- nova/tests/test_compute.py | 37 ++++++--------- nova/tests/test_console.py | 19 +++----- nova/tests/test_libvirt.py | 66 +++++++-------------------- nova/tests/test_vmwareapi.py | 15 +++---- nova/tests/test_xenapi.py | 85 ++++++++++++++++++----------------- 9 files changed, 103 insertions(+), 234 deletions(-) diff --git a/nova/tests/hyperv_unittest.py b/nova/tests/hyperv_unittest.py index ab2995923..0ea196950 100644 --- a/nova/tests/hyperv_unittest.py +++ b/nova/tests/hyperv_unittest.py @@ -23,7 +23,6 @@ from nova import context from nova import db from nova import flags from nova import test -from nova.auth import manager from nova.virt import hyperv FLAGS = flags.FLAGS @@ -34,11 +33,9 @@ class HyperVTestCase(test.TestCase): """Test cases for the Hyper-V driver""" def setUp(self): super(HyperVTestCase, self).setUp() - self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake', - admin=True) - self.project = self.manager.create_project('fake', 'fake', 'fake') - self.context = context.RequestContext(self.user.id, self.project.id) + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) def test_create_destroy(self): """Create a VM and destroy it""" diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py index f8abe609d..fde26e31a 100644 --- a/nova/tests/test_adminapi.py +++ b/nova/tests/test_adminapi.py @@ -25,7 +25,6 @@ from nova import log as logging from nova import rpc from nova import test from nova import utils -from nova.auth import manager from nova.api.ec2 import admin from nova.image import fake @@ -51,11 +50,11 @@ class AdminApiTestCase(test.TestCase): self.volume = self.start_service('volume') self.image_service = utils.import_object(FLAGS.image_service) - self.manager = manager.AuthManager() - self.user = self.manager.create_user('admin', 'admin', 'admin', True) - self.project = self.manager.create_project('proj', 'admin', 'proj') - self.context = context.RequestContext(user_id=self.user.id, - project_id=self.project.id) + self.user_id = 'admin' + self.project_id = 'admin' + self.context = context.RequestContext(self.user_id, + self.project_id, + True) def fake_show(meh, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, @@ -73,11 +72,6 @@ class AdminApiTestCase(test.TestCase): self.stubs.Set(rpc, 'cast', finish_cast) - def tearDown(self): - self.manager.delete_project(self.project) - self.manager.delete_user(self.user) - super(AdminApiTestCase, self).tearDown() - def test_block_external_ips(self): """Make sure provider firewall rules are created.""" result = self.api.block_external_addresses(self.context, '1.1.1.1/32') diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index 26ac5ff24..978e43abd 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -34,7 +34,6 @@ from nova.api import ec2 from nova.api.ec2 import apirequest from nova.api.ec2 import cloud from nova.api.ec2 import ec2utils -from nova.auth import manager class FakeHttplibSocket(object): @@ -192,10 +191,13 @@ class ApiEc2TestCase(test.TestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): super(ApiEc2TestCase, self).setUp() - self.manager = manager.AuthManager() self.host = '127.0.0.1' - self.app = ec2.Authenticate(ec2.Requestify(ec2.Executor(), - 'nova.api.ec2.cloud.CloudController')) + # NOTE(vish): skipping the Authorizer + roles = ['sysadmin', 'netadmin'] + ctxt = context.RequestContext('fake', 'fake', roles=roles) + self.app = ec2.InjectContext(ctxt, + ec2.Requestify(ec2.Authorizer(ec2.Executor()), + 'nova.api.ec2.cloud.CloudController')) def expect_http(self, host=None, is_secure=False, api_version=None): """Returns a new EC2 connection""" @@ -242,39 +244,25 @@ class ApiEc2TestCase(test.TestCase): self.expect_http(api_version='2010-10-30') self.mox.ReplayAll() - user = self.manager.create_user('fake', 'fake', 'fake') - project = self.manager.create_project('fake', 'fake', 'fake') - # Any request should be fine self.ec2.get_all_instances() self.assertTrue(self.ec2.APIVersion in self.http.getresponsebody(), 'The version in the xmlns of the response does ' 'not match the API version given in the request.') - self.manager.delete_project(project) - self.manager.delete_user(user) - def test_describe_instances(self): """Test that, after creating a user and a project, the describe instances call to the API works properly""" self.expect_http() self.mox.ReplayAll() - user = self.manager.create_user('fake', 'fake', 'fake') - project = self.manager.create_project('fake', 'fake', 'fake') self.assertEqual(self.ec2.get_all_instances(), []) - self.manager.delete_project(project) - self.manager.delete_user(user) def test_terminate_invalid_instance(self): """Attempt to terminate an invalid instance""" self.expect_http() self.mox.ReplayAll() - user = self.manager.create_user('fake', 'fake', 'fake') - project = self.manager.create_project('fake', 'fake', 'fake') self.assertRaises(EC2ResponseError, self.ec2.terminate_instances, "i-00000005") - self.manager.delete_project(project) - self.manager.delete_user(user) def test_get_all_key_pairs(self): """Test that, after creating a user and project and generating @@ -283,16 +271,12 @@ class ApiEc2TestCase(test.TestCase): self.mox.ReplayAll() keyname = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ for x in range(random.randint(4, 8))) - user = self.manager.create_user('fake', 'fake', 'fake') - project = self.manager.create_project('fake', 'fake', 'fake') # NOTE(vish): create depends on pool, so call helper directly - cloud._gen_key(context.get_admin_context(), user.id, keyname) + cloud._gen_key(context.get_admin_context(), 'fake', keyname) rv = self.ec2.get_all_key_pairs() results = [k for k in rv if k.name == keyname] self.assertEquals(len(results), 1) - self.manager.delete_project(project) - self.manager.delete_user(user) def test_create_duplicate_key_pair(self): """Test that, after successfully generating a keypair, @@ -301,8 +285,6 @@ class ApiEc2TestCase(test.TestCase): self.mox.ReplayAll() keyname = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ for x in range(random.randint(4, 8))) - user = self.manager.create_user('fake', 'fake', 'fake') - project = self.manager.create_project('fake', 'fake', 'fake') # NOTE(vish): create depends on pool, so call helper directly self.ec2.create_key_pair('test') @@ -321,27 +303,16 @@ class ApiEc2TestCase(test.TestCase): """Test that we can retrieve security groups""" self.expect_http() self.mox.ReplayAll() - user = self.manager.create_user('fake', 'fake', 'fake', admin=True) - project = self.manager.create_project('fake', 'fake', 'fake') rv = self.ec2.get_all_security_groups() self.assertEquals(len(rv), 1) self.assertEquals(rv[0].name, 'default') - self.manager.delete_project(project) - self.manager.delete_user(user) - def test_create_delete_security_group(self): """Test that we can create a security group""" self.expect_http() self.mox.ReplayAll() - user = self.manager.create_user('fake', 'fake', 'fake', admin=True) - project = self.manager.create_project('fake', 'fake', 'fake') - - # At the moment, you need both of these to actually be netadmin - self.manager.add_role('fake', 'netadmin') - project.add_role('fake', 'netadmin') security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") for x in range(random.randint(4, 8))) @@ -360,9 +331,6 @@ class ApiEc2TestCase(test.TestCase): self.ec2.delete_security_group(security_group_name) - self.manager.delete_project(project) - self.manager.delete_user(user) - def test_authorize_revoke_security_group_cidr(self): """ Test that we can add and remove CIDR based rules @@ -370,12 +338,6 @@ class ApiEc2TestCase(test.TestCase): """ self.expect_http() self.mox.ReplayAll() - user = self.manager.create_user('fake', 'fake', 'fake') - project = self.manager.create_project('fake', 'fake', 'fake') - - # At the moment, you need both of these to actually be netadmin - self.manager.add_role('fake', 'netadmin') - project.add_role('fake', 'netadmin') security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") for x in range(random.randint(4, 8))) @@ -422,9 +384,6 @@ class ApiEc2TestCase(test.TestCase): self.assertEqual(len(rv), 1) self.assertEqual(rv[0].name, 'default') - self.manager.delete_project(project) - self.manager.delete_user(user) - return def test_authorize_revoke_security_group_cidr_v6(self): @@ -434,12 +393,7 @@ class ApiEc2TestCase(test.TestCase): """ self.expect_http() self.mox.ReplayAll() - user = self.manager.create_user('fake', 'fake', 'fake') - project = self.manager.create_project('fake', 'fake', 'fake') - # At the moment, you need both of these to actually be netadmin - self.manager.add_role('fake', 'netadmin') - project.add_role('fake', 'netadmin') security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") for x in range(random.randint(4, 8))) @@ -485,9 +439,6 @@ class ApiEc2TestCase(test.TestCase): self.assertEqual(len(rv), 1) self.assertEqual(rv[0].name, 'default') - self.manager.delete_project(project) - self.manager.delete_user(user) - return def test_authorize_revoke_security_group_foreign_group(self): @@ -497,12 +448,6 @@ class ApiEc2TestCase(test.TestCase): """ self.expect_http() self.mox.ReplayAll() - user = self.manager.create_user('fake', 'fake', 'fake', admin=True) - project = self.manager.create_project('fake', 'fake', 'fake') - - # At the moment, you need both of these to actually be netadmin - self.manager.add_role('fake', 'netadmin') - project.add_role('fake', 'netadmin') rand_string = 'sdiuisudfsdcnpaqwertasd' security_group_name = "".join(random.choice(rand_string) @@ -556,8 +501,3 @@ class ApiEc2TestCase(test.TestCase): self.mox.ReplayAll() self.ec2.delete_security_group(security_group_name) - - self.manager.delete_project(project) - self.manager.delete_user(user) - - return diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 71ac7f473..c414e0ddc 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -34,7 +34,6 @@ from nova import network from nova import rpc from nova import test from nova import utils -from nova.auth import manager from nova.api.ec2 import cloud from nova.api.ec2 import ec2utils from nova.image import fake @@ -62,12 +61,11 @@ class CloudTestCase(test.TestCase): self.volume = self.start_service('volume') self.image_service = utils.import_object(FLAGS.image_service) - self.manager = manager.AuthManager() - self.user = self.manager.create_user('admin', 'admin', 'admin', True) - self.project = self.manager.create_project('proj', 'admin', 'proj') - self.context = context.RequestContext(user_id=self.user.id, - project_id=self.project.id) - host = self.network.host + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, + self.project_id, + True) def fake_show(meh, context, id): return {'id': 1, 'container_format': 'ami', @@ -87,12 +85,10 @@ class CloudTestCase(test.TestCase): self.stubs.Set(rpc, 'cast', finish_cast) def tearDown(self): - networks = db.project_get_networks(self.context, self.project.id, + networks = db.project_get_networks(self.context, self.project_id, associate=False) for network in networks: db.network_disassociate(self.context, network['id']) - self.manager.delete_project(self.project) - self.manager.delete_user(self.user) super(CloudTestCase, self).tearDown() def _create_key(self, name): diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 5d59b628a..a1b86276f 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -19,10 +19,6 @@ Tests For Compute """ -import mox -import stubout - -from nova.auth import manager from nova import compute from nova.compute import instance_types from nova.compute import manager as compute_manager @@ -67,10 +63,9 @@ class ComputeTestCase(test.TestCase): network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) self.compute_api = compute.API() - self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake') - self.project = self.manager.create_project('fake', 'fake', 'fake') - self.context = context.RequestContext('fake', 'fake', False) + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) test_notifier.NOTIFICATIONS = [] def fake_show(meh, context, id): @@ -78,19 +73,14 @@ class ComputeTestCase(test.TestCase): self.stubs.Set(nova.image.fake._FakeImageService, 'show', fake_show) - def tearDown(self): - self.manager.delete_user(self.user) - self.manager.delete_project(self.project) - super(ComputeTestCase, self).tearDown() - def _create_instance(self, params={}): """Create a test instance""" inst = {} inst['image_ref'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' - inst['user_id'] = self.user.id - inst['project_id'] = self.project.id + inst['user_id'] = self.user_id + inst['project_id'] = self.project_id type_id = instance_types.get_instance_type_by_name('m1.tiny')['id'] inst['instance_type_id'] = type_id inst['ami_launch_index'] = 0 @@ -115,8 +105,8 @@ class ComputeTestCase(test.TestCase): def _create_group(self): values = {'name': 'testgroup', 'description': 'testgroup', - 'user_id': self.user.id, - 'project_id': self.project.id} + 'user_id': self.user_id, + 'project_id': self.project_id} return db.security_group_create(self.context, values) def _get_dummy_instance(self): @@ -350,8 +340,8 @@ class ComputeTestCase(test.TestCase): self.assertEquals(msg['priority'], 'INFO') self.assertEquals(msg['event_type'], 'compute.instance.create') payload = msg['payload'] - self.assertEquals(payload['tenant_id'], self.project.id) - self.assertEquals(payload['user_id'], self.user.id) + self.assertEquals(payload['tenant_id'], self.project_id) + self.assertEquals(payload['user_id'], self.user_id) self.assertEquals(payload['instance_id'], instance_id) self.assertEquals(payload['instance_type'], 'm1.tiny') type_id = instance_types.get_instance_type_by_name('m1.tiny')['id'] @@ -374,8 +364,8 @@ class ComputeTestCase(test.TestCase): self.assertEquals(msg['priority'], 'INFO') self.assertEquals(msg['event_type'], 'compute.instance.delete') payload = msg['payload'] - self.assertEquals(payload['tenant_id'], self.project.id) - self.assertEquals(payload['user_id'], self.user.id) + self.assertEquals(payload['tenant_id'], self.project_id) + self.assertEquals(payload['user_id'], self.user_id) self.assertEquals(payload['instance_id'], instance_id) self.assertEquals(payload['instance_type'], 'm1.tiny') type_id = instance_types.get_instance_type_by_name('m1.tiny')['id'] @@ -457,8 +447,8 @@ class ComputeTestCase(test.TestCase): self.assertEquals(msg['priority'], 'INFO') self.assertEquals(msg['event_type'], 'compute.instance.resize.prep') payload = msg['payload'] - self.assertEquals(payload['tenant_id'], self.project.id) - self.assertEquals(payload['user_id'], self.user.id) + self.assertEquals(payload['tenant_id'], self.project_id) + self.assertEquals(payload['user_id'], self.user_id) self.assertEquals(payload['instance_id'], instance_id) self.assertEquals(payload['instance_type'], 'm1.tiny') type_id = instance_types.get_instance_type_by_name('m1.tiny')['id'] @@ -850,7 +840,6 @@ class ComputeTestCase(test.TestCase): def test_run_kill_vm(self): """Detect when a vm is terminated behind the scenes""" - self.stubs = stubout.StubOutForTesting() self.stubs.Set(compute_manager.ComputeManager, '_report_driver_status', nop_report_driver_status) diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 1806cc1ea..cf7f592cf 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -26,10 +26,9 @@ from nova import exception from nova import flags from nova import test from nova import utils -from nova.auth import manager -from nova.console import manager as console_manager FLAGS = flags.FLAGS +flags.DECLARE('console_driver', 'nova.console.manager') class ConsoleTestCase(test.TestCase): @@ -39,17 +38,11 @@ class ConsoleTestCase(test.TestCase): self.flags(console_driver='nova.console.fake.FakeConsoleProxy', stub_compute=True) self.console = utils.import_object(FLAGS.console_manager) - self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake') - self.project = self.manager.create_project('fake', 'fake', 'fake') - self.context = context.get_admin_context() + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) self.host = 'test_compute_host' - def tearDown(self): - self.manager.delete_user(self.user) - self.manager.delete_project(self.project) - super(ConsoleTestCase, self).tearDown() - def _create_instance(self): """Create a test instance""" inst = {} @@ -58,8 +51,8 @@ class ConsoleTestCase(test.TestCase): inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' - inst['user_id'] = self.user.id - inst['project_id'] = self.project.id + inst['user_id'] = self.user_id + inst['project_id'] = self.project_id inst['instance_type_id'] = 1 inst['ami_launch_index'] = 0 return db.instance_create(self.context, inst)['id'] diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 948ca215f..61e95c05e 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -32,7 +32,6 @@ from nova import flags from nova import test from nova import utils from nova.api.ec2 import cloud -from nova.auth import manager from nova.compute import power_state from nova.virt.libvirt import connection from nova.virt.libvirt import firewall @@ -150,35 +149,14 @@ class LibvirtConnTestCase(test.TestCase): super(LibvirtConnTestCase, self).setUp() connection._late_load_cheetah() self.flags(fake_call=True) - self.manager = manager.AuthManager() - - try: - pjs = self.manager.get_projects() - pjs = [p for p in pjs if p.name == 'fake'] - if 0 != len(pjs): - self.manager.delete_project(pjs[0]) - - users = self.manager.get_users() - users = [u for u in users if u.name == 'fake'] - if 0 != len(users): - self.manager.delete_user(users[0]) - except Exception, e: - pass - - users = self.manager.get_users() - self.user = self.manager.create_user('fake', 'fake', 'fake', - admin=True) - self.project = self.manager.create_project('fake', 'fake', 'fake') + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) self.network = utils.import_object(FLAGS.network_manager) self.context = context.get_admin_context() FLAGS.instances_path = '' self.call_libvirt_dependant_setup = False - def tearDown(self): - self.manager.delete_project(self.project) - self.manager.delete_user(self.user) - super(LibvirtConnTestCase, self).tearDown() - test_ip = '10.11.12.13' test_instance = {'memory_kb': '1024000', 'basepath': '/some/path', @@ -429,13 +407,13 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(parameters[1].get('value'), 'fake') def _check_xml_and_container(self, instance): - user_context = context.RequestContext(self.user.id, - self.project.id) + user_context = context.RequestContext(self.user_id, + self.project_id) instance_ref = db.instance_create(user_context, instance) # Re-get the instance so it's bound to an actual session instance_ref = db.instance_get(user_context, instance_ref['id']) network_ref = db.project_get_networks(context.get_admin_context(), - self.project.id)[0] + self.project_id)[0] vif = {'address': '56:12:12:12:12:12', 'network_id': network_ref['id'], @@ -475,10 +453,10 @@ class LibvirtConnTestCase(test.TestCase): def _check_xml_and_uri(self, instance, expect_ramdisk, expect_kernel, rescue=False): - user_context = context.RequestContext(self.user.id, self.project.id) + user_context = context.RequestContext(self.user_id, self.project_id) instance_ref = db.instance_create(user_context, instance) network_ref = db.project_get_networks(context.get_admin_context(), - self.project.id)[0] + self.project_id)[0] _setup_networking(instance_ref['id'], ip=self.test_ip) @@ -759,7 +737,7 @@ class LibvirtConnTestCase(test.TestCase): conn.firewall_driver.setattr('prepare_instance_filter', fake_none) network = db.project_get_networks(context.get_admin_context(), - self.project.id)[0] + self.project_id)[0] ip_dict = {'ip': self.test_ip, 'netmask': network['netmask'], 'enabled': '1'} @@ -814,11 +792,9 @@ class IptablesFirewallTestCase(test.TestCase): def setUp(self): super(IptablesFirewallTestCase, self).setUp() - self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake', - admin=True) - self.project = self.manager.create_project('fake', 'fake', 'fake') - self.context = context.RequestContext('fake', 'fake') + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) self.network = utils.import_object(FLAGS.network_manager) class FakeLibvirtConnection(object): @@ -843,11 +819,6 @@ class IptablesFirewallTestCase(test.TestCase): connection.libxml2 = __import__('libxml2') return True - def tearDown(self): - self.manager.delete_project(self.project) - self.manager.delete_user(self.user) - super(IptablesFirewallTestCase, self).tearDown() - in_nat_rules = [ '# Generated by iptables-save v1.4.10 on Sat Feb 19 00:03:19 2011', '*nat', @@ -1161,22 +1132,15 @@ class NWFilterTestCase(test.TestCase): class Mock(object): pass - self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake', - admin=True) - self.project = self.manager.create_project('fake', 'fake', 'fake') - self.context = context.RequestContext(self.user.id, self.project.id) + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) self.fake_libvirt_connection = Mock() self.fw = firewall.NWFilterFirewall( lambda: self.fake_libvirt_connection) - def tearDown(self): - self.manager.delete_project(self.project) - self.manager.delete_user(self.user) - super(NWFilterTestCase, self).tearDown() - def test_cidr_rule_nwfilter_xml(self): cloud_controller = cloud.CloudController() cloud_controller.create_security_group(self.context, diff --git a/nova/tests/test_vmwareapi.py b/nova/tests/test_vmwareapi.py index cbf7801cf..52b5debf5 100644 --- a/nova/tests/test_vmwareapi.py +++ b/nova/tests/test_vmwareapi.py @@ -26,7 +26,6 @@ from nova import db from nova import flags from nova import test from nova import utils -from nova.auth import manager from nova.compute import power_state from nova.tests.glance import stubs as glance_stubs from nova.tests.vmwareapi import db_fakes @@ -48,12 +47,10 @@ class VMWareAPIVMTestCase(test.TestCase): # self.flags(vmwareapi_host_ip='test_url', # vmwareapi_host_username='test_username', # vmwareapi_host_password='test_pass') - # self.manager = manager.AuthManager() - # self.user = self.manager.create_user('fake', 'fake', 'fake', - # admin=True) - # self.project = self.manager.create_project('fake', 'fake', 'fake') # self.network = utils.import_object(FLAGS.network_manager) - # self.stubs = stubout.StubOutForTesting() + # self.user_id = 'fake' + # self.project_id = 'fake' + # self.context = context.RequestContext(self.user_id, self.project_id) # vmwareapi_fake.reset() # db_fakes.stub_out_db_instance_api(self.stubs) # stubs.set_stubs(self.stubs) @@ -64,15 +61,13 @@ class VMWareAPIVMTestCase(test.TestCase): #def tearDown(self): # super(VMWareAPIVMTestCase, self).tearDown() # vmwareapi_fake.cleanup() - # self.manager.delete_project(self.project) - # self.manager.delete_user(self.user) # self.stubs.UnsetAll() def _create_instance_in_the_db(self): values = {'name': 1, 'id': 1, - 'project_id': self.project.id, - 'user_id': self.user.id, + 'project_id': self.project_id, + 'user_id': self.user_id, 'image_id': "1", 'kernel_id': "1", 'ramdisk_id': "1", diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 4cb7447d3..651c7f9e7 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -30,7 +30,6 @@ from nova import flags from nova import log as logging from nova import test from nova import utils -from nova.auth import manager from nova.compute import instance_types from nova.compute import power_state from nova import exception @@ -69,7 +68,9 @@ class XenAPIVolumeTestCase(test.TestCase): def setUp(self): super(XenAPIVolumeTestCase, self).setUp() self.stubs = stubout.StubOutForTesting() - self.context = context.RequestContext('fake', 'fake', False) + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) FLAGS.target_host = '127.0.0.1' FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' @@ -77,7 +78,7 @@ class XenAPIVolumeTestCase(test.TestCase): stubs.stub_out_get_target(self.stubs) xenapi_fake.reset() self.values = {'id': 1, - 'project_id': 'fake', + 'project_id': self.user_id, 'user_id': 'fake', 'image_ref': 1, 'kernel_id': 2, @@ -173,10 +174,6 @@ class XenAPIVMTestCase(test.TestCase): """Unit tests for VM operations.""" def setUp(self): super(XenAPIVMTestCase, self).setUp() - self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake', - admin=True) - self.project = self.manager.create_project('fake', 'fake', 'fake') self.network = utils.import_object(FLAGS.network_manager) self.stubs = stubout.StubOutForTesting() self.flags(xenapi_connection_url='test_url', @@ -195,7 +192,9 @@ class XenAPIVMTestCase(test.TestCase): stubs.stub_out_vm_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs) fake_utils.stub_out_utils_execute(self.stubs) - self.context = context.RequestContext('fake', 'fake', False) + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) self.conn = xenapi_conn.get_connection(False) def test_parallel_builds(self): @@ -229,8 +228,8 @@ class XenAPIVMTestCase(test.TestCase): instance = db.instance_create(self.context, values) self.conn.spawn(instance, network_info) - gt1 = eventlet.spawn(_do_build, 1, self.project.id, self.user.id) - gt2 = eventlet.spawn(_do_build, 2, self.project.id, self.user.id) + gt1 = eventlet.spawn(_do_build, 1, self.project_id, self.user_id) + gt2 = eventlet.spawn(_do_build, 2, self.project_id, self.user_id) gt1.wait() gt2.wait() @@ -399,8 +398,8 @@ class XenAPIVMTestCase(test.TestCase): check_injection=False): stubs.stubout_loopingcall_start(self.stubs) values = {'id': instance_id, - 'project_id': self.project.id, - 'user_id': self.user.id, + 'project_id': self.project_id, + 'user_id': self.user_id, 'image_ref': image_ref, 'kernel_id': kernel_id, 'ramdisk_id': ramdisk_id, @@ -465,12 +464,30 @@ class XenAPIVMTestCase(test.TestCase): self._check_vdis(vdi_recs_start, vdi_recs_end) def test_spawn_raw_objectstore(self): - FLAGS.xenapi_image_service = 'objectstore' - self._test_spawn(1, None, None) + # TODO(vish): deprecated + from nova.auth import manager + authman = manager.AuthManager() + authman.create_user('fake', 'fake') + authman.create_project('fake', 'fake') + try: + FLAGS.xenapi_image_service = 'objectstore' + self._test_spawn(1, None, None) + finally: + authman.delete_project('fake') + authman.delete_user('fake') def test_spawn_objectstore(self): - FLAGS.xenapi_image_service = 'objectstore' - self._test_spawn(1, 2, 3) + # TODO(vish): deprecated + from nova.auth import manager + authman = manager.AuthManager() + authman.create_user('fake', 'fake') + authman.create_project('fake', 'fake') + try: + FLAGS.xenapi_image_service = 'objectstore' + self._test_spawn(1, 2, 3) + finally: + authman.delete_project('fake') + authman.delete_user('fake') @stub_vm_utils_with_vdi_attached_here def test_spawn_raw_glance(self): @@ -599,7 +616,7 @@ class XenAPIVMTestCase(test.TestCase): # guest agent is detected self.assertFalse(self._tee_executed) - @test.skip_test("Never gets an address, not sure why") + @test.skip_test("Key Error on domid") def test_spawn_vlanmanager(self): self.flags(xenapi_image_service='glance', network_manager='nova.network.manager.VlanManager', @@ -609,7 +626,7 @@ class XenAPIVMTestCase(test.TestCase): def dummy(*args, **kwargs): pass - self.stubs.Set(VMOps, 'create_vifs', dummy) + self.stubs.Set(vmops.VMOps, 'create_vifs', dummy) # Reset network table xenapi_fake.reset_table('network') # Instance id = 2 will use vlan network (see db/fakes.py) @@ -623,7 +640,7 @@ class XenAPIVMTestCase(test.TestCase): self.network.set_network_host(ctxt, network['id']) self.network.allocate_for_instance(ctxt, instance_id=instance_ref.id, - instance_type_id=1, project_id=self.project.id) + instance_type_id=1, project_id=self.project_id) self.network.setup_compute_network(ctxt, instance_ref.id) self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, @@ -655,21 +672,13 @@ class XenAPIVMTestCase(test.TestCase): # Ensure that it will not unrescue a non-rescued instance. self.assertRaises(Exception, conn.unrescue, instance, None) - def tearDown(self): - super(XenAPIVMTestCase, self).tearDown() - self.manager.delete_project(self.project) - self.manager.delete_user(self.user) - self.vm_info = None - self.vm = None - self.stubs.UnsetAll() - def _create_instance(self, instance_id=1): """Creates and spawns a test instance.""" stubs.stubout_loopingcall_start(self.stubs) values = { 'id': instance_id, - 'project_id': self.project.id, - 'user_id': self.user.id, + 'project_id': self.project_id, + 'user_id': self.user_id, 'image_ref': 1, 'kernel_id': 2, 'ramdisk_id': 3, @@ -750,14 +759,12 @@ class XenAPIMigrateInstance(test.TestCase): stubs.stub_out_get_target(self.stubs) xenapi_fake.reset() xenapi_fake.create_network('fake', FLAGS.flat_network_bridge) - self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake', - admin=True) - self.project = self.manager.create_project('fake', 'fake', 'fake') - self.context = context.RequestContext('fake', 'fake', False) + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) self.values = {'id': 1, - 'project_id': self.project.id, - 'user_id': self.user.id, + 'project_id': self.project_id, + 'user_id': self.user_id, 'image_ref': 1, 'kernel_id': None, 'ramdisk_id': None, @@ -771,12 +778,6 @@ class XenAPIMigrateInstance(test.TestCase): stubs.stubout_get_this_vm_uuid(self.stubs) glance_stubs.stubout_glance_client(self.stubs) - def tearDown(self): - super(XenAPIMigrateInstance, self).tearDown() - self.manager.delete_project(self.project) - self.manager.delete_user(self.user) - self.stubs.UnsetAll() - def test_migrate_disk_and_power_off(self): instance = db.instance_create(self.context, self.values) stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) From c200d953079f179f03303600b86585c1c804c2a0 Mon Sep 17 00:00:00 2001 From: Nikolay Sokolov Date: Fri, 22 Jul 2011 17:26:11 +0400 Subject: [PATCH 07/91] Moved restaring instances from livbirt driver to ComputeManager. --- nova/flags.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/flags.py b/nova/flags.py index 49355b436..23ca38b17 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -387,3 +387,6 @@ DEFINE_list('zone_capabilities', 'Key/Multi-value list representng capabilities of this zone') DEFINE_string('build_plan_encryption_key', None, '128bit (hex) encryption key for scheduler build plans.') + +DEFINE_bool('start_guests_on_host_boot', False, + 'Whether to restart guests when the host reboots') From e5509f7766fd8fe7c82540f5910c6e6d7979c9a2 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 22 Jul 2011 19:47:41 +0000 Subject: [PATCH 08/91] fix all tests --- nova/tests/test_access.py | 2 +- nova/tests/test_api.py | 3 ++- nova/wsgi.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_access.py b/nova/tests/test_access.py index 6069c5d71..39558b1cf 100644 --- a/nova/tests/test_access.py +++ b/nova/tests/test_access.py @@ -41,7 +41,7 @@ class FakeApiRequest(object): class AccessTestCase(test.TestCase): def _env_for(self, ctxt, action): env = {} - env['ec2.context'] = ctxt + env['nova.context'] = ctxt env['ec2.request'] = FakeApiRequest(action) return env diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index 978e43abd..292f9d668 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -30,6 +30,7 @@ import webob from nova import context from nova import exception from nova import test +from nova import wsgi from nova.api import ec2 from nova.api.ec2 import apirequest from nova.api.ec2 import cloud @@ -195,7 +196,7 @@ class ApiEc2TestCase(test.TestCase): # NOTE(vish): skipping the Authorizer roles = ['sysadmin', 'netadmin'] ctxt = context.RequestContext('fake', 'fake', roles=roles) - self.app = ec2.InjectContext(ctxt, + self.app = wsgi.InjectContext(ctxt, ec2.Requestify(ec2.Authorizer(ec2.Executor()), 'nova.api.ec2.cloud.CloudController')) diff --git a/nova/wsgi.py b/nova/wsgi.py index eae3afcb4..c8ddb97d7 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -274,6 +274,18 @@ class Middleware(Application): return self.process_response(response) +class InjectContext(Middleware): + """Add a 'nova.context' to WSGI environ.""" + def __init__(self, context, *args, **kwargs): + self.context = context + super(InjectContext, self).__init__(*args, **kwargs) + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + req.environ['nova.context'] = self.context + return self.application + + class Debug(Middleware): """Helper class for debugging a WSGI application. From 62b9e10bc09834f0c577a71251c9c92ff1ed0c86 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 22 Jul 2011 20:20:31 +0000 Subject: [PATCH 09/91] fix auth tests --- nova/auth/manager.py | 2 +- nova/tests/test_auth.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index b6131fb7f..06af7e781 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -785,7 +785,7 @@ class AuthManager(object): return read_buffer def get_environment_rc(self, user, project=None, use_dmz=True): - """Get credential zip for user in project""" + """Get environment rc for user in project""" if not isinstance(user, User): user = self.get_user(user) if project is None: diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 71e0d17c9..7c0f783bb 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -102,7 +102,7 @@ class _AuthManagerBaseTestCase(test.TestCase): self.assertEqual('classified', u.secret) self.assertEqual('private-party', u.access) - def test_004_signature_is_valid(self): + def test_signature_is_valid(self): with user_generator(self.manager, name='admin', secret='admin', access='admin'): with project_generator(self.manager, name="admin", @@ -141,15 +141,14 @@ class _AuthManagerBaseTestCase(test.TestCase): '127.0.0.1', '/services/Cloud')) - def test_005_can_get_credentials(self): - return - credentials = self.manager.get_user('test1').get_credentials() - self.assertEqual(credentials, - 'export EC2_ACCESS_KEY="access"\n' + - 'export EC2_SECRET_KEY="secret"\n' + - 'export EC2_URL="http://127.0.0.1:8773/services/Cloud"\n' + - 'export S3_URL="http://127.0.0.1:3333/"\n' + - 'export EC2_USER_ID="test1"\n') + def test_can_get_credentials(self): + st = {'access': 'access', 'secret': 'secret'} + with user_and_project_generator(self.manager, user_state=st) as (u, p): + credentials = self.manager.get_environment_rc(u, p) + LOG.debug(credentials) + self.assertTrue('export EC2_ACCESS_KEY="access:testproj"\n' + in credentials) + self.assertTrue('export EC2_SECRET_KEY="secret"\n' in credentials) def test_can_list_users(self): with user_generator(self.manager): From bc9f1f47b59a57384236fdc1f4d851d9534a7b80 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 22 Jul 2011 20:41:46 +0000 Subject: [PATCH 10/91] fix test_access --- nova/auth/manager.py | 9 +++++++++ nova/tests/test_access.py | 19 +++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 06af7e781..7f99d9016 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -518,6 +518,15 @@ class AuthManager(object): return drv.get_user_roles(User.safe_id(user), Project.safe_id(project)) + def get_active_roles(self, user, project=None): + """Get all active roles for context""" + if project: + roles = FLAGS.allowed_roles + roles.append('projectmanager') + else: + roles = FLAGS.global_roles + return [role for role in roles if self.has_role(user, role, project)] + def get_project(self, pid): """Get project object by id""" with self.driver() as drv: diff --git a/nova/tests/test_access.py b/nova/tests/test_access.py index 39558b1cf..3b54fc249 100644 --- a/nova/tests/test_access.py +++ b/nova/tests/test_access.py @@ -16,7 +16,6 @@ # License for the specific language governing permissions and limitations # under the License. -import unittest import webob from nova import context @@ -93,7 +92,11 @@ class AccessTestCase(test.TestCase): super(AccessTestCase, self).tearDown() def response_status(self, user, methodName): - ctxt = context.RequestContext(user.id, self.project.id) + roles = manager.AuthManager().get_active_roles(user, self.project) + ctxt = context.RequestContext(user.id, + self.project.id, + is_admin=user.is_admin(), + roles=roles) environ = self._env_for(ctxt, methodName) req = webob.Request.blank('/', environ) resp = req.get_response(self.mw) @@ -105,30 +108,26 @@ class AccessTestCase(test.TestCase): def shouldDeny(self, user, methodName): self.assertEqual(401, self.response_status(user, methodName)) - def test_001_allow_all(self): + def test_allow_all(self): users = [self.testadmin, self.testpmsys, self.testnet, self.testsys] for user in users: self.shouldAllow(user, '_allow_all') - def test_002_allow_none(self): + def test_allow_none(self): self.shouldAllow(self.testadmin, '_allow_none') users = [self.testpmsys, self.testnet, self.testsys] for user in users: self.shouldDeny(user, '_allow_none') - def test_003_allow_project_manager(self): + def test_allow_project_manager(self): for user in [self.testadmin, self.testpmsys]: self.shouldAllow(user, '_allow_project_manager') for user in [self.testnet, self.testsys]: self.shouldDeny(user, '_allow_project_manager') - def test_004_allow_sys_and_net(self): + def test_allow_sys_and_net(self): for user in [self.testadmin, self.testnet, self.testsys]: self.shouldAllow(user, '_allow_sys_and_net') # denied because it doesn't have the per project sysadmin for user in [self.testpmsys]: self.shouldDeny(user, '_allow_sys_and_net') - -if __name__ == "__main__": - # TODO: Implement use_fake as an option - unittest.main() From 2bd2ddf211f649e88a74f8954855ec9370e5d685 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 22 Jul 2011 21:36:41 +0000 Subject: [PATCH 11/91] clean up fake auth manager in other places --- nova/auth/manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 7f99d9016..5118abba2 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -521,8 +521,7 @@ class AuthManager(object): def get_active_roles(self, user, project=None): """Get all active roles for context""" if project: - roles = FLAGS.allowed_roles - roles.append('projectmanager') + roles = FLAGS.allowed_roles + ['projectmanager'] else: roles = FLAGS.global_roles return [role for role in roles if self.has_role(user, role, project)] From 9465f6b87c6cadf57a71345ce9e0865791336d84 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 22 Jul 2011 22:04:52 +0000 Subject: [PATCH 12/91] remove auth manager from instance helper --- nova/auth/manager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 5118abba2..6205cfb56 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -738,10 +738,6 @@ class AuthManager(object): with self.driver() as drv: drv.modify_user(uid, access_key, secret_key, admin) - @staticmethod - def get_key_pairs(context): - return db.key_pair_get_all_by_user(context.elevated(), context.user_id) - def get_credentials(self, user, project=None, use_dmz=True): """Get credential zip for user in project""" if not isinstance(user, User): From 910fa5c94c102055d14b386d50dfb84ea23b49e3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 22 Jul 2011 22:12:22 +0000 Subject: [PATCH 13/91] pep cleanup --- nova/tests/test_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index 292f9d668..3ec1c9abf 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -395,7 +395,6 @@ class ApiEc2TestCase(test.TestCase): self.expect_http() self.mox.ReplayAll() - security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") for x in range(random.randint(4, 8))) From 736163c747f679b889dfa5b00010ddbca8f2dcc3 Mon Sep 17 00:00:00 2001 From: Nikolay Sokolov Date: Tue, 26 Jul 2011 00:31:42 +0400 Subject: [PATCH 14/91] Fixed old libvirt semantics, added resume_guests_state_on_host_boot flag. --- nova/flags.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/flags.py b/nova/flags.py index 23ca38b17..6c7e448ad 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -390,3 +390,5 @@ DEFINE_string('build_plan_encryption_key', None, DEFINE_bool('start_guests_on_host_boot', False, 'Whether to restart guests when the host reboots') +DEFINE_bool('resume_guests_state_on_host_boot', False, + 'Whether to start guests, that was running before the host reboot') From 0b607dd897782728211231622f7ffd04591d2365 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Mon, 25 Jul 2011 22:55:37 -0400 Subject: [PATCH 15/91] removing xenapi_image_service flag --- nova/tests/test_xenapi.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 4cb7447d3..87e2e93b3 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -429,7 +429,7 @@ class XenAPIVMTestCase(test.TestCase): self.assertTrue(instance.architecture) def test_spawn_not_enough_memory(self): - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.assertRaises(Exception, self._test_spawn, 1, 2, 3, "4") # m1.xlarge @@ -441,7 +441,7 @@ class XenAPIVMTestCase(test.TestCase): """ vdi_recs_start = self._list_vdis() - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' stubs.stubout_fetch_image_glance_disk(self.stubs) self.assertRaises(xenapi_fake.Failure, self._test_spawn, 1, 2, 3) @@ -456,7 +456,7 @@ class XenAPIVMTestCase(test.TestCase): """ vdi_recs_start = self._list_vdis() - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' stubs.stubout_create_vm(self.stubs) self.assertRaises(xenapi_fake.Failure, self._test_spawn, 1, 2, 3) @@ -465,21 +465,21 @@ class XenAPIVMTestCase(test.TestCase): self._check_vdis(vdi_recs_start, vdi_recs_end) def test_spawn_raw_objectstore(self): - FLAGS.xenapi_image_service = 'objectstore' + FLAGS.image_service = 'nova.image.s3.S3ImageService' self._test_spawn(1, None, None) def test_spawn_objectstore(self): - FLAGS.xenapi_image_service = 'objectstore' + FLAGS.image_service = 'nova.image.s3.S3ImageService' self._test_spawn(1, 2, 3) @stub_vm_utils_with_vdi_attached_here def test_spawn_raw_glance(self): - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_RAW, None, None) self.check_vm_params_for_linux() def test_spawn_vhd_glance_linux(self): - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, os_type="linux", architecture="x86-64") self.check_vm_params_for_linux() @@ -508,20 +508,20 @@ class XenAPIVMTestCase(test.TestCase): self.assertEqual(len(self.vm['VBDs']), 1) def test_spawn_vhd_glance_windows(self): - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, os_type="windows", architecture="i386") self.check_vm_params_for_windows() def test_spawn_glance(self): - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, glance_stubs.FakeGlance.IMAGE_RAMDISK) self.check_vm_params_for_linux_with_external_kernel() def test_spawn_netinject_file(self): - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' db_fakes.stub_out_db_instance_api(self.stubs, injected=True) self._tee_executed = False @@ -547,7 +547,7 @@ class XenAPIVMTestCase(test.TestCase): # Capture the sudo tee .../etc/network/interfaces command (r'(sudo\s+)?tee.*interfaces', _tee_handler), ]) - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, glance_stubs.FakeGlance.IMAGE_RAMDISK, @@ -555,7 +555,7 @@ class XenAPIVMTestCase(test.TestCase): self.assertTrue(self._tee_executed) def test_spawn_netinject_xenstore(self): - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' db_fakes.stub_out_db_instance_api(self.stubs, injected=True) self._tee_executed = False @@ -601,7 +601,7 @@ class XenAPIVMTestCase(test.TestCase): @test.skip_test("Never gets an address, not sure why") def test_spawn_vlanmanager(self): - self.flags(xenapi_image_service='glance', + self.flags(image_service='nova.image.glance.GlanceImageService', network_manager='nova.network.manager.VlanManager', network_driver='nova.network.xenapi_net', vlan_interface='fake0') @@ -784,6 +784,7 @@ class XenAPIMigrateInstance(test.TestCase): conn.migrate_disk_and_power_off(instance, '127.0.0.1') def test_finish_resize(self): + FLAGS.image_service = 'nova.image.glance.GlanceImageService' instance = db.instance_create(self.context, self.values) stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) stubs.stubout_loopingcall_start(self.stubs) @@ -827,7 +828,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): def test_instance_disk(self): """If a kernel is specified, the image type is DISK (aka machine).""" - FLAGS.xenapi_image_service = 'objectstore' + FLAGS.image_service = 'nova.image.s3.S3ImageService' self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_MACHINE self.fake_instance.kernel_id = glance_stubs.FakeGlance.IMAGE_KERNEL self.assert_disk_type(vm_utils.ImageType.DISK) @@ -837,7 +838,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): If the kernel isn't specified, and we're not using Glance, then DISK_RAW is assumed. """ - FLAGS.xenapi_image_service = 'objectstore' + FLAGS.image_service = 'nova.image.s3.S3ImageService' self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_RAW self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_RAW) @@ -847,7 +848,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): If we're using Glance, then defer to the image_type field, which in this case will be 'raw'. """ - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_RAW self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_RAW) @@ -857,7 +858,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): If we're using Glance, then defer to the image_type field, which in this case will be 'vhd'. """ - FLAGS.xenapi_image_service = 'glance' + FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_VHD self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_VHD) From 5f313c23a9d7a741a17677f5aacdde4bf043e40b Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Tue, 26 Jul 2011 11:26:33 -0400 Subject: [PATCH 16/91] removing objectstore and image_service flag checking --- nova/tests/test_xenapi.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 87e2e93b3..77d3062be 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -429,7 +429,6 @@ class XenAPIVMTestCase(test.TestCase): self.assertTrue(instance.architecture) def test_spawn_not_enough_memory(self): - FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.assertRaises(Exception, self._test_spawn, 1, 2, 3, "4") # m1.xlarge @@ -441,7 +440,6 @@ class XenAPIVMTestCase(test.TestCase): """ vdi_recs_start = self._list_vdis() - FLAGS.image_service = 'nova.image.glance.GlanceImageService' stubs.stubout_fetch_image_glance_disk(self.stubs) self.assertRaises(xenapi_fake.Failure, self._test_spawn, 1, 2, 3) @@ -456,7 +454,6 @@ class XenAPIVMTestCase(test.TestCase): """ vdi_recs_start = self._list_vdis() - FLAGS.image_service = 'nova.image.glance.GlanceImageService' stubs.stubout_create_vm(self.stubs) self.assertRaises(xenapi_fake.Failure, self._test_spawn, 1, 2, 3) @@ -464,22 +461,12 @@ class XenAPIVMTestCase(test.TestCase): vdi_recs_end = self._list_vdis() self._check_vdis(vdi_recs_start, vdi_recs_end) - def test_spawn_raw_objectstore(self): - FLAGS.image_service = 'nova.image.s3.S3ImageService' - self._test_spawn(1, None, None) - - def test_spawn_objectstore(self): - FLAGS.image_service = 'nova.image.s3.S3ImageService' - self._test_spawn(1, 2, 3) - @stub_vm_utils_with_vdi_attached_here def test_spawn_raw_glance(self): - FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_RAW, None, None) self.check_vm_params_for_linux() def test_spawn_vhd_glance_linux(self): - FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, os_type="linux", architecture="x86-64") self.check_vm_params_for_linux() @@ -508,20 +495,17 @@ class XenAPIVMTestCase(test.TestCase): self.assertEqual(len(self.vm['VBDs']), 1) def test_spawn_vhd_glance_windows(self): - FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, os_type="windows", architecture="i386") self.check_vm_params_for_windows() def test_spawn_glance(self): - FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, glance_stubs.FakeGlance.IMAGE_RAMDISK) self.check_vm_params_for_linux_with_external_kernel() def test_spawn_netinject_file(self): - FLAGS.image_service = 'nova.image.glance.GlanceImageService' db_fakes.stub_out_db_instance_api(self.stubs, injected=True) self._tee_executed = False @@ -547,7 +531,6 @@ class XenAPIVMTestCase(test.TestCase): # Capture the sudo tee .../etc/network/interfaces command (r'(sudo\s+)?tee.*interfaces', _tee_handler), ]) - FLAGS.image_service = 'nova.image.glance.GlanceImageService' self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, glance_stubs.FakeGlance.IMAGE_RAMDISK, @@ -555,7 +538,6 @@ class XenAPIVMTestCase(test.TestCase): self.assertTrue(self._tee_executed) def test_spawn_netinject_xenstore(self): - FLAGS.image_service = 'nova.image.glance.GlanceImageService' db_fakes.stub_out_db_instance_api(self.stubs, injected=True) self._tee_executed = False @@ -784,7 +766,6 @@ class XenAPIMigrateInstance(test.TestCase): conn.migrate_disk_and_power_off(instance, '127.0.0.1') def test_finish_resize(self): - FLAGS.image_service = 'nova.image.glance.GlanceImageService' instance = db.instance_create(self.context, self.values) stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) stubs.stubout_loopingcall_start(self.stubs) @@ -828,7 +809,6 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): def test_instance_disk(self): """If a kernel is specified, the image type is DISK (aka machine).""" - FLAGS.image_service = 'nova.image.s3.S3ImageService' self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_MACHINE self.fake_instance.kernel_id = glance_stubs.FakeGlance.IMAGE_KERNEL self.assert_disk_type(vm_utils.ImageType.DISK) @@ -838,7 +818,6 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): If the kernel isn't specified, and we're not using Glance, then DISK_RAW is assumed. """ - FLAGS.image_service = 'nova.image.s3.S3ImageService' self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_RAW self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_RAW) @@ -848,7 +827,6 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): If we're using Glance, then defer to the image_type field, which in this case will be 'raw'. """ - FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_RAW self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_RAW) @@ -858,7 +836,6 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): If we're using Glance, then defer to the image_type field, which in this case will be 'vhd'. """ - FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_VHD self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_VHD) From 7d0c8b163bb211d4571773fc04e19b26cc1d4029 Mon Sep 17 00:00:00 2001 From: John Tran Date: Tue, 26 Jul 2011 10:03:16 -0700 Subject: [PATCH 17/91] added warning when size of subnet(s) being created are larger than FLAG.network_size in attempt to alleviate confusion. For example, currently when 'nova-manage network create foo 192.168.0.0/16', the result is that it creates a 192.168.0.0/24 instead without any indication to why. --- bin/nova-manage | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bin/nova-manage b/bin/nova-manage index b63bd326f..da9538e39 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -56,6 +56,7 @@ import gettext import glob import json +import math import netaddr import os import sys @@ -669,6 +670,14 @@ class NetworkCommands(object): num_networks = FLAGS.num_networks if not network_size: network_size = FLAGS.network_size + fixnet = netaddr.IPNetwork(fixed_range) + each_subnet_size = fixnet.size / int(num_networks) + if each_subnet_size > network_size: + subnet = 32 - int(math.log(network_size, 2)) + oversize_msg = _('Subnet(s) too large, defaulting to /%s.' + ' To override, specify network_size flag.' + % subnet) + print oversize_msg if not multi_host: multi_host = FLAGS.multi_host else: From 766d22833abbe019184ff4994176f8fd799bf2db Mon Sep 17 00:00:00 2001 From: John Tran Date: Tue, 26 Jul 2011 13:12:34 -0700 Subject: [PATCH 18/91] fixed per peer review --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index da9538e39..ca60d28d6 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -676,7 +676,7 @@ class NetworkCommands(object): subnet = 32 - int(math.log(network_size, 2)) oversize_msg = _('Subnet(s) too large, defaulting to /%s.' ' To override, specify network_size flag.' - % subnet) + ) % subnet print oversize_msg if not multi_host: multi_host = FLAGS.multi_host From 444801ca4d359e037fdcc2bd125c4fd5d11e665c Mon Sep 17 00:00:00 2001 From: Zed Shaw Date: Tue, 26 Jul 2011 16:29:50 -0700 Subject: [PATCH 19/91] Implements a simplified messaging abstraction with the least amount of impact to the code base. --- nova/rpc.py | 598 ++-------------------------------- nova/rpc_backends/__init__.py | 0 nova/rpc_backends/amqp.py | 591 +++++++++++++++++++++++++++++++++ nova/rpc_backends/common.py | 23 ++ nova/service.py | 28 +- nova/test.py | 16 - nova/tests/test_adminapi.py | 2 +- nova/tests/test_cloud.py | 68 +--- nova/tests/test_rpc.py | 61 +--- nova/tests/test_rpc_amqp.py | 68 ++++ nova/utils.py | 11 + 11 files changed, 751 insertions(+), 715 deletions(-) create mode 100644 nova/rpc_backends/__init__.py create mode 100644 nova/rpc_backends/amqp.py create mode 100644 nova/rpc_backends/common.py create mode 100644 nova/tests/test_rpc_amqp.py diff --git a/nova/rpc.py b/nova/rpc.py index e2771ca88..8b0c6df67 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -16,597 +16,51 @@ # License for the specific language governing permissions and limitations # under the License. -"""AMQP-based RPC. -Queues have consumers and publishers. - -No fan-out support yet. - -""" - -import json -import sys -import time -import traceback -import types -import uuid - -from carrot import connection as carrot_connection -from carrot import messaging -from eventlet import greenpool -from eventlet import pools -from eventlet import queue -import greenlet - -from nova import context -from nova import exception -from nova import fakerabbit +from nova.utils import load_module +from nova.rpc_backends.common import RemoteError, LOG from nova import flags -from nova import log as logging -from nova import utils - - -LOG = logging.getLogger('nova.rpc') - FLAGS = flags.FLAGS -flags.DEFINE_integer('rpc_thread_pool_size', 1024, - 'Size of RPC thread pool') -flags.DEFINE_integer('rpc_conn_pool_size', 30, - 'Size of RPC connection pool') +flags.DEFINE_string('rpc_backend', + 'nova.rpc_backends.amqp', + "The messaging module to use, defaults to AMQP.") +RPCIMPL = load_module(FLAGS.rpc_backend) -class Connection(carrot_connection.BrokerConnection): - """Connection instance object.""" - @classmethod - def instance(cls, new=True): - """Returns the instance.""" - if new or not hasattr(cls, '_instance'): - params = dict(hostname=FLAGS.rabbit_host, - port=FLAGS.rabbit_port, - ssl=FLAGS.rabbit_use_ssl, - userid=FLAGS.rabbit_userid, - password=FLAGS.rabbit_password, - virtual_host=FLAGS.rabbit_virtual_host) +def create_connection(new=True): + return RPCIMPL.Connection.instance(new=True) - if FLAGS.fake_rabbit: - params['backend_cls'] = fakerabbit.Backend - # NOTE(vish): magic is fun! - # pylint: disable=W0142 - if new: - return cls(**params) - else: - cls._instance = cls(**params) - return cls._instance +def create_consumer(conn, topic, proxy, fanout=False): + if fanout: + return RPCIMPL.FanoutAdapterConsumer( + connection=conn, + topic=topic, + proxy=proxy) + else: + return RPCIMPL.TopicAdapterConsumer( + connection=conn, + topic=topic, + proxy=proxy) - @classmethod - def recreate(cls): - """Recreates the connection instance. - This is necessary to recover from some network errors/disconnects. - - """ - try: - del cls._instance - except AttributeError, e: - # The _instance stuff is for testing purposes. Usually we don't use - # it. So don't freak out if it doesn't exist. - pass - return cls.instance() - - -class Pool(pools.Pool): - """Class that implements a Pool of Connections.""" - - # TODO(comstud): Timeout connections not used in a while - def create(self): - LOG.debug('Creating new connection') - return Connection.instance(new=True) - -# Create a ConnectionPool to use for RPC calls. We'll order the -# pool as a stack (LIFO), so that we can potentially loop through and -# timeout old unused connections at some point -ConnectionPool = Pool( - max_size=FLAGS.rpc_conn_pool_size, - order_as_stack=True) - - -class Consumer(messaging.Consumer): - """Consumer base class. - - Contains methods for connecting the fetch method to async loops. - - """ - - def __init__(self, *args, **kwargs): - for i in xrange(FLAGS.rabbit_max_retries): - if i > 0: - time.sleep(FLAGS.rabbit_retry_interval) - try: - super(Consumer, self).__init__(*args, **kwargs) - self.failed_connection = False - break - except Exception as e: # Catching all because carrot sucks - fl_host = FLAGS.rabbit_host - fl_port = FLAGS.rabbit_port - fl_intv = FLAGS.rabbit_retry_interval - LOG.error(_('AMQP server on %(fl_host)s:%(fl_port)d is' - ' unreachable: %(e)s. Trying again in %(fl_intv)d' - ' seconds.') % locals()) - self.failed_connection = True - if self.failed_connection: - LOG.error(_('Unable to connect to AMQP server ' - 'after %d tries. Shutting down.'), - FLAGS.rabbit_max_retries) - sys.exit(1) - - def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): - """Wraps the parent fetch with some logic for failed connection.""" - # TODO(vish): the logic for failed connections and logging should be - # refactored into some sort of connection manager object - try: - if self.failed_connection: - # NOTE(vish): connection is defined in the parent class, we can - # recreate it as long as we create the backend too - # pylint: disable=W0201 - self.connection = Connection.recreate() - self.backend = self.connection.create_backend() - self.declare() - return super(Consumer, self).fetch(no_ack, - auto_ack, - enable_callbacks) - if self.failed_connection: - LOG.error(_('Reconnected to queue')) - self.failed_connection = False - # NOTE(vish): This is catching all errors because we really don't - # want exceptions to be logged 10 times a second if some - # persistent failure occurs. - except Exception, e: # pylint: disable=W0703 - if not self.failed_connection: - LOG.exception(_('Failed to fetch message from queue: %s' % e)) - self.failed_connection = True - - def attach_to_eventlet(self): - """Only needed for unit tests!""" - timer = utils.LoopingCall(self.fetch, enable_callbacks=True) - timer.start(0.1) - return timer - - -class AdapterConsumer(Consumer): - """Calls methods on a proxy object based on method and args.""" - - def __init__(self, connection=None, topic='broadcast', proxy=None): - LOG.debug(_('Initing the Adapter Consumer for %s') % topic) - self.proxy = proxy - self.pool = greenpool.GreenPool(FLAGS.rpc_thread_pool_size) - super(AdapterConsumer, self).__init__(connection=connection, - topic=topic) - self.register_callback(self.process_data) - - def process_data(self, message_data, message): - """Consumer callback to call a method on a proxy object. - - Parses the message for validity and fires off a thread to call the - proxy object method. - - Message data should be a dictionary with two keys: - method: string representing the method to call - args: dictionary of arg: value - - Example: {'method': 'echo', 'args': {'value': 42}} - - """ - LOG.debug(_('received %s') % message_data) - # This will be popped off in _unpack_context - msg_id = message_data.get('_msg_id', None) - ctxt = _unpack_context(message_data) - - method = message_data.get('method') - args = message_data.get('args', {}) - message.ack() - if not method: - # NOTE(vish): we may not want to ack here, but that means that bad - # messages stay in the queue indefinitely, so for now - # we just log the message and send an error string - # back to the caller - LOG.warn(_('no method for message: %s') % message_data) - if msg_id: - msg_reply(msg_id, - _('No method for message: %s') % message_data) - return - self.pool.spawn_n(self._process_data, msg_id, ctxt, method, args) - - @exception.wrap_exception() - def _process_data(self, msg_id, ctxt, method, args): - """Thread that maigcally looks for a method on the proxy - object and calls it. - """ - - node_func = getattr(self.proxy, str(method)) - node_args = dict((str(k), v) for k, v in args.iteritems()) - # NOTE(vish): magic is fun! - try: - rval = node_func(context=ctxt, **node_args) - if msg_id: - # Check if the result was a generator - if isinstance(rval, types.GeneratorType): - for x in rval: - msg_reply(msg_id, x, None) - else: - msg_reply(msg_id, rval, None) - - # This final None tells multicall that it is done. - msg_reply(msg_id, None, None) - elif isinstance(rval, types.GeneratorType): - # NOTE(vish): this iterates through the generator - list(rval) - except Exception as e: - logging.exception('Exception during message handling') - if msg_id: - msg_reply(msg_id, None, sys.exc_info()) - return - - -class TopicAdapterConsumer(AdapterConsumer): - """Consumes messages on a specific topic.""" - - exchange_type = 'topic' - - def __init__(self, connection=None, topic='broadcast', proxy=None): - self.queue = topic - self.routing_key = topic - self.exchange = FLAGS.control_exchange - self.durable = False - super(TopicAdapterConsumer, self).__init__(connection=connection, - topic=topic, proxy=proxy) - - -class FanoutAdapterConsumer(AdapterConsumer): - """Consumes messages from a fanout exchange.""" - - exchange_type = 'fanout' - - def __init__(self, connection=None, topic='broadcast', proxy=None): - self.exchange = '%s_fanout' % topic - self.routing_key = topic - unique = uuid.uuid4().hex - self.queue = '%s_fanout_%s' % (topic, unique) - self.durable = False - # Fanout creates unique queue names, so we should auto-remove - # them when done, so they're not left around on restart. - # Also, we're the only one that should be consuming. exclusive - # implies auto_delete, so we'll just set that.. - self.exclusive = True - LOG.info(_('Created "%(exchange)s" fanout exchange ' - 'with "%(key)s" routing key'), - dict(exchange=self.exchange, key=self.routing_key)) - super(FanoutAdapterConsumer, self).__init__(connection=connection, - topic=topic, proxy=proxy) - - -class ConsumerSet(object): - """Groups consumers to listen on together on a single connection.""" - - def __init__(self, connection, consumer_list): - self.consumer_list = set(consumer_list) - self.consumer_set = None - self.enabled = True - self.init(connection) - - def init(self, conn): - if not conn: - conn = Connection.instance(new=True) - if self.consumer_set: - self.consumer_set.close() - self.consumer_set = messaging.ConsumerSet(conn) - for consumer in self.consumer_list: - consumer.connection = conn - # consumer.backend is set for us - self.consumer_set.add_consumer(consumer) - - def reconnect(self): - self.init(None) - - def wait(self, limit=None): - running = True - while running: - it = self.consumer_set.iterconsume(limit=limit) - if not it: - break - while True: - try: - it.next() - except StopIteration: - return - except greenlet.GreenletExit: - running = False - break - except Exception as e: - LOG.exception(_("Exception while processing consumer")) - self.reconnect() - # Break to outer loop - break - - def close(self): - self.consumer_set.close() - - -class Publisher(messaging.Publisher): - """Publisher base class.""" - pass - - -class TopicPublisher(Publisher): - """Publishes messages on a specific topic.""" - - exchange_type = 'topic' - - def __init__(self, connection=None, topic='broadcast'): - self.routing_key = topic - self.exchange = FLAGS.control_exchange - self.durable = False - super(TopicPublisher, self).__init__(connection=connection) - - -class FanoutPublisher(Publisher): - """Publishes messages to a fanout exchange.""" - - exchange_type = 'fanout' - - def __init__(self, topic, connection=None): - self.exchange = '%s_fanout' % topic - self.queue = '%s_fanout' % topic - self.durable = False - self.auto_delete = True - LOG.info(_('Creating "%(exchange)s" fanout exchange'), - dict(exchange=self.exchange)) - super(FanoutPublisher, self).__init__(connection=connection) - - -class DirectConsumer(Consumer): - """Consumes messages directly on a channel specified by msg_id.""" - - exchange_type = 'direct' - - def __init__(self, connection=None, msg_id=None): - self.queue = msg_id - self.routing_key = msg_id - self.exchange = msg_id - self.auto_delete = True - self.exclusive = True - super(DirectConsumer, self).__init__(connection=connection) - - -class DirectPublisher(Publisher): - """Publishes messages directly on a channel specified by msg_id.""" - - exchange_type = 'direct' - - def __init__(self, connection=None, msg_id=None): - self.routing_key = msg_id - self.exchange = msg_id - self.auto_delete = True - super(DirectPublisher, self).__init__(connection=connection) - - -def msg_reply(msg_id, reply=None, failure=None): - """Sends a reply or an error on the channel signified by msg_id. - - Failure should be a sys.exc_info() tuple. - - """ - if failure: - message = str(failure[1]) - tb = traceback.format_exception(*failure) - LOG.error(_("Returning exception %s to caller"), message) - LOG.error(tb) - failure = (failure[0].__name__, str(failure[1]), tb) - - with ConnectionPool.item() as conn: - publisher = DirectPublisher(connection=conn, msg_id=msg_id) - try: - publisher.send({'result': reply, 'failure': failure}) - except TypeError: - publisher.send( - {'result': dict((k, repr(v)) - for k, v in reply.__dict__.iteritems()), - 'failure': failure}) - - publisher.close() - - -class RemoteError(exception.Error): - """Signifies that a remote class has raised an exception. - - Containes a string representation of the type of the original exception, - the value of the original exception, and the traceback. These are - sent to the parent as a joined string so printing the exception - contains all of the relevent info. - - """ - - def __init__(self, exc_type, value, traceback): - self.exc_type = exc_type - self.value = value - self.traceback = traceback - super(RemoteError, self).__init__('%s %s\n%s' % (exc_type, - value, - traceback)) - - -def _unpack_context(msg): - """Unpack context from msg.""" - context_dict = {} - for key in list(msg.keys()): - # NOTE(vish): Some versions of python don't like unicode keys - # in kwargs. - key = str(key) - if key.startswith('_context_'): - value = msg.pop(key) - context_dict[key[9:]] = value - context_dict['msg_id'] = msg.pop('_msg_id', None) - LOG.debug(_('unpacked context: %s'), context_dict) - return RpcContext.from_dict(context_dict) - - -def _pack_context(msg, context): - """Pack context into msg. - - Values for message keys need to be less than 255 chars, so we pull - context out into a bunch of separate keys. If we want to support - more arguments in rabbit messages, we may want to do the same - for args at some point. - - """ - context_d = dict([('_context_%s' % key, value) - for (key, value) in context.to_dict().iteritems()]) - msg.update(context_d) - - -class RpcContext(context.RequestContext): - def __init__(self, *args, **kwargs): - msg_id = kwargs.pop('msg_id', None) - self.msg_id = msg_id - super(RpcContext, self).__init__(*args, **kwargs) - - def reply(self, *args, **kwargs): - msg_reply(self.msg_id, *args, **kwargs) - - -def multicall(context, topic, msg): - """Make a call that returns multiple times.""" - LOG.debug(_('Making asynchronous call on %s ...'), topic) - msg_id = uuid.uuid4().hex - msg.update({'_msg_id': msg_id}) - LOG.debug(_('MSG_ID is %s') % (msg_id)) - _pack_context(msg, context) - - con_conn = ConnectionPool.get() - consumer = DirectConsumer(connection=con_conn, msg_id=msg_id) - wait_msg = MulticallWaiter(consumer) - consumer.register_callback(wait_msg) - - publisher = TopicPublisher(connection=con_conn, topic=topic) - publisher.send(msg) - publisher.close() - - return wait_msg - - -class MulticallWaiter(object): - def __init__(self, consumer): - self._consumer = consumer - self._results = queue.Queue() - self._closed = False - - def close(self): - self._closed = True - self._consumer.close() - ConnectionPool.put(self._consumer.connection) - - def __call__(self, data, message): - """Acks message and sets result.""" - message.ack() - if data['failure']: - self._results.put(RemoteError(*data['failure'])) - else: - self._results.put(data['result']) - - def __iter__(self): - return self.wait() - - def wait(self): - while True: - rv = None - while rv is None and not self._closed: - try: - rv = self._consumer.fetch(enable_callbacks=True) - except Exception: - self.close() - raise - time.sleep(0.01) - - result = self._results.get() - if isinstance(result, Exception): - self.close() - raise result - if result == None: - self.close() - raise StopIteration - yield result +def create_consumer_set(conn, consumers): + return RPCIMPL.ConsumerSet(connection=conn, consumer_list=consumers) def call(context, topic, msg): - """Sends a message on a topic and wait for a response.""" - rv = multicall(context, topic, msg) - # NOTE(vish): return the last result from the multicall - rv = list(rv) - if not rv: - return - return rv[-1] + return RPCIMPL.call(context, topic, msg) def cast(context, topic, msg): - """Sends a message on a topic without waiting for a response.""" - LOG.debug(_('Making asynchronous cast on %s...'), topic) - _pack_context(msg, context) - with ConnectionPool.item() as conn: - publisher = TopicPublisher(connection=conn, topic=topic) - publisher.send(msg) - publisher.close() + return RPCIMPL.cast(context, topic, msg) def fanout_cast(context, topic, msg): - """Sends a message on a fanout exchange without waiting for a response.""" - LOG.debug(_('Making asynchronous fanout cast...')) - _pack_context(msg, context) - with ConnectionPool.item() as conn: - publisher = FanoutPublisher(topic, connection=conn) - publisher.send(msg) - publisher.close() + return RPCIMPL.fanout_cast(context, topic, msg) -def generic_response(message_data, message): - """Logs a result and exits.""" - LOG.debug(_('response %s'), message_data) - message.ack() - sys.exit(0) - - -def send_message(topic, message, wait=True): - """Sends a message for testing.""" - msg_id = uuid.uuid4().hex - message.update({'_msg_id': msg_id}) - LOG.debug(_('topic is %s'), topic) - LOG.debug(_('message %s'), message) - - if wait: - consumer = messaging.Consumer(connection=Connection.instance(), - queue=msg_id, - exchange=msg_id, - auto_delete=True, - exchange_type='direct', - routing_key=msg_id) - consumer.register_callback(generic_response) - - publisher = messaging.Publisher(connection=Connection.instance(), - exchange=FLAGS.control_exchange, - durable=False, - exchange_type='topic', - routing_key=topic) - publisher.send(message) - publisher.close() - - if wait: - consumer.wait() - consumer.close() - - -if __name__ == '__main__': - # You can send messages from the command line using - # topic and a json string representing a dictionary - # for the method - send_message(sys.argv[1], json.loads(sys.argv[2])) +def multicall(context, topic, msg): + return RPCIMPL.multicall(context, topic, msg) diff --git a/nova/rpc_backends/__init__.py b/nova/rpc_backends/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nova/rpc_backends/amqp.py b/nova/rpc_backends/amqp.py new file mode 100644 index 000000000..efa178bd2 --- /dev/null +++ b/nova/rpc_backends/amqp.py @@ -0,0 +1,591 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""AMQP-based RPC. + +Queues have consumers and publishers. + +No fan-out support yet. + +""" + +import json +import sys +import time +import traceback +import types +import uuid + +from carrot import connection as carrot_connection +from carrot import messaging +from eventlet import greenpool +from eventlet import pools +from eventlet import queue +import greenlet + +from nova import context +from nova import exception +from nova import fakerabbit +from nova import flags +from nova import log as logging +from nova import utils +from nova.rpc_backends.common import RemoteError, LOG + + +FLAGS = flags.FLAGS +flags.DEFINE_integer('rpc_thread_pool_size', 1024, + 'Size of RPC thread pool') +flags.DEFINE_integer('rpc_conn_pool_size', 30, + 'Size of RPC connection pool') + + +class Connection(carrot_connection.BrokerConnection): + """Connection instance object.""" + + @classmethod + def instance(cls, new=True): + """Returns the instance.""" + if new or not hasattr(cls, '_instance'): + params = dict(hostname=FLAGS.rabbit_host, + port=FLAGS.rabbit_port, + ssl=FLAGS.rabbit_use_ssl, + userid=FLAGS.rabbit_userid, + password=FLAGS.rabbit_password, + virtual_host=FLAGS.rabbit_virtual_host) + + if FLAGS.fake_rabbit: + params['backend_cls'] = fakerabbit.Backend + + # NOTE(vish): magic is fun! + # pylint: disable=W0142 + if new: + return cls(**params) + else: + cls._instance = cls(**params) + return cls._instance + + @classmethod + def recreate(cls): + """Recreates the connection instance. + + This is necessary to recover from some network errors/disconnects. + + """ + try: + del cls._instance + except AttributeError, e: + # The _instance stuff is for testing purposes. Usually we don't use + # it. So don't freak out if it doesn't exist. + pass + return cls.instance() + + +class Pool(pools.Pool): + """Class that implements a Pool of Connections.""" + + # TODO(comstud): Timeout connections not used in a while + def create(self): + LOG.debug('Creating new connection') + return Connection.instance(new=True) + +# Create a ConnectionPool to use for RPC calls. We'll order the +# pool as a stack (LIFO), so that we can potentially loop through and +# timeout old unused connections at some point +ConnectionPool = Pool( + max_size=FLAGS.rpc_conn_pool_size, + order_as_stack=True) + + +class Consumer(messaging.Consumer): + """Consumer base class. + + Contains methods for connecting the fetch method to async loops. + + """ + + def __init__(self, *args, **kwargs): + for i in xrange(FLAGS.rabbit_max_retries): + if i > 0: + time.sleep(FLAGS.rabbit_retry_interval) + try: + super(Consumer, self).__init__(*args, **kwargs) + self.failed_connection = False + break + except Exception as e: # Catching all because carrot sucks + fl_host = FLAGS.rabbit_host + fl_port = FLAGS.rabbit_port + fl_intv = FLAGS.rabbit_retry_interval + LOG.error(_('AMQP server on %(fl_host)s:%(fl_port)d is' + ' unreachable: %(e)s. Trying again in %(fl_intv)d' + ' seconds.') % locals()) + self.failed_connection = True + if self.failed_connection: + LOG.error(_('Unable to connect to AMQP server ' + 'after %d tries. Shutting down.'), + FLAGS.rabbit_max_retries) + sys.exit(1) + + def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): + """Wraps the parent fetch with some logic for failed connection.""" + # TODO(vish): the logic for failed connections and logging should be + # refactored into some sort of connection manager object + try: + if self.failed_connection: + # NOTE(vish): connection is defined in the parent class, we can + # recreate it as long as we create the backend too + # pylint: disable=W0201 + self.connection = Connection.recreate() + self.backend = self.connection.create_backend() + self.declare() + return super(Consumer, self).fetch(no_ack, + auto_ack, + enable_callbacks) + if self.failed_connection: + LOG.error(_('Reconnected to queue')) + self.failed_connection = False + # NOTE(vish): This is catching all errors because we really don't + # want exceptions to be logged 10 times a second if some + # persistent failure occurs. + except Exception, e: # pylint: disable=W0703 + if not self.failed_connection: + LOG.exception(_('Failed to fetch message from queue: %s' % e)) + self.failed_connection = True + + def attach_to_eventlet(self): + """Only needed for unit tests!""" + timer = utils.LoopingCall(self.fetch, enable_callbacks=True) + timer.start(0.1) + return timer + + +class AdapterConsumer(Consumer): + """Calls methods on a proxy object based on method and args.""" + + def __init__(self, connection=None, topic='broadcast', proxy=None): + LOG.debug(_('Initing the Adapter Consumer for %s') % topic) + self.proxy = proxy + self.pool = greenpool.GreenPool(FLAGS.rpc_thread_pool_size) + super(AdapterConsumer, self).__init__(connection=connection, + topic=topic) + self.register_callback(self.process_data) + + def process_data(self, message_data, message): + """Consumer callback to call a method on a proxy object. + + Parses the message for validity and fires off a thread to call the + proxy object method. + + Message data should be a dictionary with two keys: + method: string representing the method to call + args: dictionary of arg: value + + Example: {'method': 'echo', 'args': {'value': 42}} + + """ + LOG.debug(_('received %s') % message_data) + # This will be popped off in _unpack_context + msg_id = message_data.get('_msg_id', None) + ctxt = _unpack_context(message_data) + + method = message_data.get('method') + args = message_data.get('args', {}) + message.ack() + if not method: + # NOTE(vish): we may not want to ack here, but that means that bad + # messages stay in the queue indefinitely, so for now + # we just log the message and send an error string + # back to the caller + LOG.warn(_('no method for message: %s') % message_data) + if msg_id: + msg_reply(msg_id, + _('No method for message: %s') % message_data) + return + self.pool.spawn_n(self._process_data, msg_id, ctxt, method, args) + + @exception.wrap_exception() + def _process_data(self, msg_id, ctxt, method, args): + """Thread that maigcally looks for a method on the proxy + object and calls it. + """ + + node_func = getattr(self.proxy, str(method)) + node_args = dict((str(k), v) for k, v in args.iteritems()) + # NOTE(vish): magic is fun! + try: + rval = node_func(context=ctxt, **node_args) + if msg_id: + # Check if the result was a generator + if isinstance(rval, types.GeneratorType): + for x in rval: + msg_reply(msg_id, x, None) + else: + msg_reply(msg_id, rval, None) + + # This final None tells multicall that it is done. + msg_reply(msg_id, None, None) + elif isinstance(rval, types.GeneratorType): + # NOTE(vish): this iterates through the generator + list(rval) + except Exception as e: + logging.exception('Exception during message handling') + if msg_id: + msg_reply(msg_id, None, sys.exc_info()) + return + + +class TopicAdapterConsumer(AdapterConsumer): + """Consumes messages on a specific topic.""" + + exchange_type = 'topic' + + def __init__(self, connection=None, topic='broadcast', proxy=None): + self.queue = topic + self.routing_key = topic + self.exchange = FLAGS.control_exchange + self.durable = False + super(TopicAdapterConsumer, self).__init__(connection=connection, + topic=topic, proxy=proxy) + + +class FanoutAdapterConsumer(AdapterConsumer): + """Consumes messages from a fanout exchange.""" + + exchange_type = 'fanout' + + def __init__(self, connection=None, topic='broadcast', proxy=None): + self.exchange = '%s_fanout' % topic + self.routing_key = topic + unique = uuid.uuid4().hex + self.queue = '%s_fanout_%s' % (topic, unique) + self.durable = False + # Fanout creates unique queue names, so we should auto-remove + # them when done, so they're not left around on restart. + # Also, we're the only one that should be consuming. exclusive + # implies auto_delete, so we'll just set that.. + self.exclusive = True + LOG.info(_('Created "%(exchange)s" fanout exchange ' + 'with "%(key)s" routing key'), + dict(exchange=self.exchange, key=self.routing_key)) + super(FanoutAdapterConsumer, self).__init__(connection=connection, + topic=topic, proxy=proxy) + + +class ConsumerSet(object): + """Groups consumers to listen on together on a single connection.""" + + def __init__(self, connection, consumer_list): + self.consumer_list = set(consumer_list) + self.consumer_set = None + self.enabled = True + self.init(connection) + + def init(self, conn): + if not conn: + conn = Connection.instance(new=True) + if self.consumer_set: + self.consumer_set.close() + self.consumer_set = messaging.ConsumerSet(conn) + for consumer in self.consumer_list: + consumer.connection = conn + # consumer.backend is set for us + self.consumer_set.add_consumer(consumer) + + def reconnect(self): + self.init(None) + + def wait(self, limit=None): + running = True + while running: + it = self.consumer_set.iterconsume(limit=limit) + if not it: + break + while True: + try: + it.next() + except StopIteration: + return + except greenlet.GreenletExit: + running = False + break + except Exception as e: + LOG.exception(_("Exception while processing consumer")) + self.reconnect() + # Break to outer loop + break + + def close(self): + self.consumer_set.close() + + +class Publisher(messaging.Publisher): + """Publisher base class.""" + pass + + +class TopicPublisher(Publisher): + """Publishes messages on a specific topic.""" + + exchange_type = 'topic' + + def __init__(self, connection=None, topic='broadcast'): + self.routing_key = topic + self.exchange = FLAGS.control_exchange + self.durable = False + super(TopicPublisher, self).__init__(connection=connection) + + +class FanoutPublisher(Publisher): + """Publishes messages to a fanout exchange.""" + + exchange_type = 'fanout' + + def __init__(self, topic, connection=None): + self.exchange = '%s_fanout' % topic + self.queue = '%s_fanout' % topic + self.durable = False + self.auto_delete = True + LOG.info(_('Creating "%(exchange)s" fanout exchange'), + dict(exchange=self.exchange)) + super(FanoutPublisher, self).__init__(connection=connection) + + +class DirectConsumer(Consumer): + """Consumes messages directly on a channel specified by msg_id.""" + + exchange_type = 'direct' + + def __init__(self, connection=None, msg_id=None): + self.queue = msg_id + self.routing_key = msg_id + self.exchange = msg_id + self.auto_delete = True + self.exclusive = True + super(DirectConsumer, self).__init__(connection=connection) + + +class DirectPublisher(Publisher): + """Publishes messages directly on a channel specified by msg_id.""" + + exchange_type = 'direct' + + def __init__(self, connection=None, msg_id=None): + self.routing_key = msg_id + self.exchange = msg_id + self.auto_delete = True + super(DirectPublisher, self).__init__(connection=connection) + + +def msg_reply(msg_id, reply=None, failure=None): + """Sends a reply or an error on the channel signified by msg_id. + + Failure should be a sys.exc_info() tuple. + + """ + if failure: + message = str(failure[1]) + tb = traceback.format_exception(*failure) + LOG.error(_("Returning exception %s to caller"), message) + LOG.error(tb) + failure = (failure[0].__name__, str(failure[1]), tb) + + with ConnectionPool.item() as conn: + publisher = DirectPublisher(connection=conn, msg_id=msg_id) + try: + publisher.send({'result': reply, 'failure': failure}) + except TypeError: + publisher.send( + {'result': dict((k, repr(v)) + for k, v in reply.__dict__.iteritems()), + 'failure': failure}) + + publisher.close() + + +def _unpack_context(msg): + """Unpack context from msg.""" + context_dict = {} + for key in list(msg.keys()): + # NOTE(vish): Some versions of python don't like unicode keys + # in kwargs. + key = str(key) + if key.startswith('_context_'): + value = msg.pop(key) + context_dict[key[9:]] = value + context_dict['msg_id'] = msg.pop('_msg_id', None) + LOG.debug(_('unpacked context: %s'), context_dict) + return RpcContext.from_dict(context_dict) + + +def _pack_context(msg, context): + """Pack context into msg. + + Values for message keys need to be less than 255 chars, so we pull + context out into a bunch of separate keys. If we want to support + more arguments in rabbit messages, we may want to do the same + for args at some point. + + """ + context_d = dict([('_context_%s' % key, value) + for (key, value) in context.to_dict().iteritems()]) + msg.update(context_d) + + +class RpcContext(context.RequestContext): + def __init__(self, *args, **kwargs): + msg_id = kwargs.pop('msg_id', None) + self.msg_id = msg_id + super(RpcContext, self).__init__(*args, **kwargs) + + def reply(self, *args, **kwargs): + msg_reply(self.msg_id, *args, **kwargs) + + +def multicall(context, topic, msg): + """Make a call that returns multiple times.""" + LOG.debug(_('Making asynchronous call on %s ...'), topic) + msg_id = uuid.uuid4().hex + msg.update({'_msg_id': msg_id}) + LOG.debug(_('MSG_ID is %s') % (msg_id)) + _pack_context(msg, context) + + con_conn = ConnectionPool.get() + consumer = DirectConsumer(connection=con_conn, msg_id=msg_id) + wait_msg = MulticallWaiter(consumer) + consumer.register_callback(wait_msg) + + publisher = TopicPublisher(connection=con_conn, topic=topic) + publisher.send(msg) + publisher.close() + + return wait_msg + + +class MulticallWaiter(object): + def __init__(self, consumer): + self._consumer = consumer + self._results = queue.Queue() + self._closed = False + + def close(self): + self._closed = True + self._consumer.close() + ConnectionPool.put(self._consumer.connection) + + def __call__(self, data, message): + """Acks message and sets result.""" + message.ack() + if data['failure']: + self._results.put(RemoteError(*data['failure'])) + else: + self._results.put(data['result']) + + def __iter__(self): + return self.wait() + + def wait(self): + while True: + rv = None + while rv is None and not self._closed: + try: + rv = self._consumer.fetch(enable_callbacks=True) + except Exception: + self.close() + raise + time.sleep(0.01) + + result = self._results.get() + if isinstance(result, Exception): + self.close() + raise result + if result == None: + self.close() + raise StopIteration + yield result + + +def call(context, topic, msg): + """Sends a message on a topic and wait for a response.""" + rv = multicall(context, topic, msg) + # NOTE(vish): return the last result from the multicall + rv = list(rv) + if not rv: + return + return rv[-1] + + +def cast(context, topic, msg): + """Sends a message on a topic without waiting for a response.""" + LOG.debug(_('Making asynchronous cast on %s...'), topic) + _pack_context(msg, context) + with ConnectionPool.item() as conn: + publisher = TopicPublisher(connection=conn, topic=topic) + publisher.send(msg) + publisher.close() + + +def fanout_cast(context, topic, msg): + """Sends a message on a fanout exchange without waiting for a response.""" + LOG.debug(_('Making asynchronous fanout cast...')) + _pack_context(msg, context) + with ConnectionPool.item() as conn: + publisher = FanoutPublisher(topic, connection=conn) + publisher.send(msg) + publisher.close() + + +def generic_response(message_data, message): + """Logs a result and exits.""" + LOG.debug(_('response %s'), message_data) + message.ack() + sys.exit(0) + + +def send_message(topic, message, wait=True): + """Sends a message for testing.""" + msg_id = uuid.uuid4().hex + message.update({'_msg_id': msg_id}) + LOG.debug(_('topic is %s'), topic) + LOG.debug(_('message %s'), message) + + if wait: + consumer = messaging.Consumer(connection=Connection.instance(), + queue=msg_id, + exchange=msg_id, + auto_delete=True, + exchange_type='direct', + routing_key=msg_id) + consumer.register_callback(generic_response) + + publisher = messaging.Publisher(connection=Connection.instance(), + exchange=FLAGS.control_exchange, + durable=False, + exchange_type='topic', + routing_key=topic) + publisher.send(message) + publisher.close() + + if wait: + consumer.wait() + consumer.close() + + +if __name__ == '__main__': + # You can send messages from the command line using + # topic and a json string representing a dictionary + # for the method + send_message(sys.argv[1], json.loads(sys.argv[2])) diff --git a/nova/rpc_backends/common.py b/nova/rpc_backends/common.py new file mode 100644 index 000000000..1d3065a83 --- /dev/null +++ b/nova/rpc_backends/common.py @@ -0,0 +1,23 @@ +from nova import exception +from nova import log as logging + +LOG = logging.getLogger('nova.rpc') + + +class RemoteError(exception.Error): + """Signifies that a remote class has raised an exception. + + Containes a string representation of the type of the original exception, + the value of the original exception, and the traceback. These are + sent to the parent as a joined string so printing the exception + contains all of the relevent info. + + """ + + def __init__(self, exc_type, value, traceback): + self.exc_type = exc_type + self.value = value + self.traceback = traceback + super(RemoteError, self).__init__('%s %s\n%s' % (exc_type, + value, + traceback)) diff --git a/nova/service.py b/nova/service.py index 00e4f61e5..6e9eddc5a 100644 --- a/nova/service.py +++ b/nova/service.py @@ -149,26 +149,22 @@ class Service(object): if 'nova-compute' == self.binary: self.manager.update_available_resource(ctxt) - self.conn = rpc.Connection.instance(new=True) + self.conn = rpc.create_connection(new=True) logging.debug("Creating Consumer connection for Service %s" % self.topic) # Share this same connection for these Consumers - consumer_all = rpc.TopicAdapterConsumer( - connection=self.conn, - topic=self.topic, - proxy=self) - consumer_node = rpc.TopicAdapterConsumer( - connection=self.conn, - topic='%s.%s' % (self.topic, self.host), - proxy=self) - fanout = rpc.FanoutAdapterConsumer( - connection=self.conn, - topic=self.topic, - proxy=self) - consumer_set = rpc.ConsumerSet( - connection=self.conn, - consumer_list=[consumer_all, consumer_node, fanout]) + consumer_all = rpc.create_consumer(self.conn, self.topic, self, + fanout=False) + + node_topic = '%s.%s' % (self.topic, self.host) + consumer_node = rpc.create_consumer(self.conn, node_topic, self, + fanout=False) + + fanout = rpc.create_consumer(self.conn, self.topic, self, fanout=True) + + consumers = [consumer_all, consumer_node, fanout] + consumer_set = rpc.create_consumer_set(self.conn, consumers) # Wait forever, processing these consumers def _wait(): diff --git a/nova/test.py b/nova/test.py index 9790b0aa1..549aa6fcf 100644 --- a/nova/test.py +++ b/nova/test.py @@ -99,9 +99,7 @@ class TestCase(unittest.TestCase): self.flag_overrides = {} self.injected = [] self._services = [] - self._monkey_patch_attach() self._original_flags = FLAGS.FlagValuesDict() - rpc.ConnectionPool = rpc.Pool(max_size=FLAGS.rpc_conn_pool_size) def tearDown(self): """Runs after each test method to tear down test environment.""" @@ -126,9 +124,6 @@ class TestCase(unittest.TestCase): # Reset any overriden flags self.reset_flags() - # Reset our monkey-patches - rpc.Consumer.attach_to_eventlet = self.original_attach - # Stop any timers for x in self.injected: try: @@ -172,17 +167,6 @@ class TestCase(unittest.TestCase): self._services.append(svc) return svc - def _monkey_patch_attach(self): - self.original_attach = rpc.Consumer.attach_to_eventlet - - def _wrapped(inner_self): - rv = self.original_attach(inner_self) - self.injected.append(rv) - return rv - - _wrapped.func_name = self.original_attach.func_name - rpc.Consumer.attach_to_eventlet = _wrapped - # Useful assertions def assertDictMatch(self, d1, d2, approx_equal=False, tolerance=0.001): """Assert two dicts are equivalent. diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py index 877cf4ea1..6bbe15f53 100644 --- a/nova/tests/test_adminapi.py +++ b/nova/tests/test_adminapi.py @@ -39,7 +39,7 @@ class AdminApiTestCase(test.TestCase): super(AdminApiTestCase, self).setUp() self.flags(connection_type='fake') - self.conn = rpc.Connection.instance() + self.conn = rpc.create_connection() # set up our cloud self.api = admin.AdminController() diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 136082cc1..a1b296a96 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -50,7 +50,7 @@ class CloudTestCase(test.TestCase): self.flags(connection_type='fake', stub_network=True) - self.conn = rpc.Connection.instance() + self.conn = rpc.create_connection() # set up our cloud self.cloud = cloud.CloudController() @@ -269,63 +269,24 @@ class CloudTestCase(test.TestCase): delete = self.cloud.delete_security_group self.assertRaises(exception.ApiError, delete, self.context) - def test_authorize_security_group_ingress(self): + def test_authorize_revoke_security_group_ingress(self): kwargs = {'project_id': self.context.project_id, 'name': 'test'} sec = db.security_group_create(self.context, kwargs) authz = self.cloud.authorize_security_group_ingress kwargs = {'to_port': '999', 'from_port': '999', 'ip_protocol': 'tcp'} - self.assertTrue(authz(self.context, group_name=sec['name'], **kwargs)) - - def test_authorize_security_group_ingress_ip_permissions_ip_ranges(self): - kwargs = {'project_id': self.context.project_id, 'name': 'test'} - sec = db.security_group_create(self.context, kwargs) - authz = self.cloud.authorize_security_group_ingress - kwargs = {'ip_permissions': [{'to_port': 81, 'from_port': 81, - 'ip_ranges': - {'1': {'cidr_ip': u'0.0.0.0/0'}, - '2': {'cidr_ip': u'10.10.10.10/32'}}, - 'ip_protocol': u'tcp'}]} - self.assertTrue(authz(self.context, group_name=sec['name'], **kwargs)) - - def test_authorize_security_group_ingress_ip_permissions_groups(self): - kwargs = {'project_id': self.context.project_id, 'name': 'test'} - sec = db.security_group_create(self.context, kwargs) - authz = self.cloud.authorize_security_group_ingress - kwargs = {'ip_permissions': [{'to_port': 81, 'from_port': 81, - 'ip_ranges':{'1': {'cidr_ip': u'0.0.0.0/0'}, - '2': {'cidr_ip': u'10.10.10.10/32'}}, - 'groups': {'1': {'user_id': u'someuser', - 'group_name': u'somegroup1'}, - '2': {'user_id': u'someuser', - 'group_name': u'othergroup2'}}, - 'ip_protocol': u'tcp'}]} - self.assertTrue(authz(self.context, group_name=sec['name'], **kwargs)) - - def test_revoke_security_group_ingress(self): - kwargs = {'project_id': self.context.project_id, 'name': 'test'} - sec = db.security_group_create(self.context, kwargs) - authz = self.cloud.authorize_security_group_ingress - kwargs = {'to_port': '999', 'from_port': '999', 'ip_protocol': 'tcp'} - authz(self.context, group_id=sec['id'], **kwargs) + authz(self.context, group_name=sec['name'], **kwargs) revoke = self.cloud.revoke_security_group_ingress self.assertTrue(revoke(self.context, group_name=sec['name'], **kwargs)) - def test_revoke_security_group_ingress_by_id(self): - kwargs = {'project_id': self.context.project_id, 'name': 'test'} - sec = db.security_group_create(self.context, kwargs) - authz = self.cloud.authorize_security_group_ingress - kwargs = {'to_port': '999', 'from_port': '999', 'ip_protocol': 'tcp'} - authz(self.context, group_id=sec['id'], **kwargs) - revoke = self.cloud.revoke_security_group_ingress - self.assertTrue(revoke(self.context, group_id=sec['id'], **kwargs)) - - def test_authorize_security_group_ingress_by_id(self): + def test_authorize_revoke_security_group_ingress_by_id(self): sec = db.security_group_create(self.context, {'project_id': self.context.project_id, 'name': 'test'}) authz = self.cloud.authorize_security_group_ingress kwargs = {'to_port': '999', 'from_port': '999', 'ip_protocol': 'tcp'} - self.assertTrue(authz(self.context, group_id=sec['id'], **kwargs)) + authz(self.context, group_id=sec['id'], **kwargs) + revoke = self.cloud.revoke_security_group_ingress + self.assertTrue(revoke(self.context, group_id=sec['id'], **kwargs)) def test_authorize_security_group_ingress_missing_protocol_params(self): sec = db.security_group_create(self.context, @@ -947,21 +908,6 @@ class CloudTestCase(test.TestCase): self._wait_for_running(ec2_instance_id) return ec2_instance_id - def test_rescue_unrescue_instance(self): - instance_id = self._run_instance( - image_id='ami-1', - instance_type=FLAGS.default_instance_type, - max_count=1) - self.cloud.rescue_instance(context=self.context, - instance_id=instance_id) - # NOTE(vish): This currently does no validation, it simply makes sure - # that the code path doesn't throw an exception. - self.cloud.unrescue_instance(context=self.context, - instance_id=instance_id) - # TODO(soren): We need this until we can stop polling in the rpc code - # for unit tests. - self.cloud.terminate_instances(self.context, [instance_id]) - def test_console_output(self): instance_id = self._run_instance( image_id='ami-1', diff --git a/nova/tests/test_rpc.py b/nova/tests/test_rpc.py index ffd748efe..2d2436175 100644 --- a/nova/tests/test_rpc.py +++ b/nova/tests/test_rpc.py @@ -33,11 +33,12 @@ LOG = logging.getLogger('nova.tests.rpc') class RpcTestCase(test.TestCase): def setUp(self): super(RpcTestCase, self).setUp() - self.conn = rpc.Connection.instance(True) + self.conn = rpc.create_connection(True) self.receiver = TestReceiver() - self.consumer = rpc.TopicAdapterConsumer(connection=self.conn, - topic='test', - proxy=self.receiver) + self.consumer = rpc.create_consumer(self.conn, + 'test', + self.receiver, + False) self.consumer.attach_to_eventlet() self.context = context.get_admin_context() @@ -129,6 +130,8 @@ class RpcTestCase(test.TestCase): """Calls echo in the passed queue""" LOG.debug(_("Nested received %(queue)s, %(value)s") % locals()) + # TODO: so, it will replay the context and use the same REQID? + # that's bizarre. ret = rpc.call(context, queue, {"method": "echo", @@ -137,10 +140,11 @@ class RpcTestCase(test.TestCase): return value nested = Nested() - conn = rpc.Connection.instance(True) - consumer = rpc.TopicAdapterConsumer(connection=conn, - topic='nested', - proxy=nested) + conn = rpc.create_connection(True) + consumer = rpc.create_consumer(conn, + 'nested', + nested, + False) consumer.attach_to_eventlet() value = 42 result = rpc.call(self.context, @@ -149,47 +153,6 @@ class RpcTestCase(test.TestCase): "value": value}}) self.assertEqual(value, result) - def test_connectionpool_single(self): - """Test that ConnectionPool recycles a single connection.""" - conn1 = rpc.ConnectionPool.get() - rpc.ConnectionPool.put(conn1) - conn2 = rpc.ConnectionPool.get() - rpc.ConnectionPool.put(conn2) - self.assertEqual(conn1, conn2) - - def test_connectionpool_double(self): - """Test that ConnectionPool returns and reuses separate connections. - - When called consecutively we should get separate connections and upon - returning them those connections should be reused for future calls - before generating a new connection. - - """ - conn1 = rpc.ConnectionPool.get() - conn2 = rpc.ConnectionPool.get() - - self.assertNotEqual(conn1, conn2) - rpc.ConnectionPool.put(conn1) - rpc.ConnectionPool.put(conn2) - - conn3 = rpc.ConnectionPool.get() - conn4 = rpc.ConnectionPool.get() - self.assertEqual(conn1, conn3) - self.assertEqual(conn2, conn4) - - def test_connectionpool_limit(self): - """Test connection pool limit and connection uniqueness.""" - max_size = FLAGS.rpc_conn_pool_size - conns = [] - - for i in xrange(max_size): - conns.append(rpc.ConnectionPool.get()) - - self.assertFalse(rpc.ConnectionPool.free_items) - self.assertEqual(rpc.ConnectionPool.current_size, - rpc.ConnectionPool.max_size) - self.assertEqual(len(set(conns)), max_size) - class TestReceiver(object): """Simple Proxy class so the consumer has methods to call. diff --git a/nova/tests/test_rpc_amqp.py b/nova/tests/test_rpc_amqp.py new file mode 100644 index 000000000..e3df2393a --- /dev/null +++ b/nova/tests/test_rpc_amqp.py @@ -0,0 +1,68 @@ +from nova import context +from nova import flags +from nova import log as logging +from nova import rpc +from nova.rpc_backends import amqp +from nova import test + + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.tests.rpc') + + +class RpcAMQPTestCase(test.TestCase): + def setUp(self): + super(RpcAMQPTestCase, self).setUp() + self.conn = rpc.create_connection(True) + self.receiver = TestReceiver() + self.consumer = rpc.create_consumer(self.conn, + 'test', + self.receiver, + False) + self.consumer.attach_to_eventlet() + self.context = context.get_admin_context() + + def test_connectionpool_single(self): + """Test that ConnectionPool recycles a single connection.""" + conn1 = amqp.ConnectionPool.get() + amqp.ConnectionPool.put(conn1) + conn2 = amqp.ConnectionPool.get() + amqp.ConnectionPool.put(conn2) + self.assertEqual(conn1, conn2) + + +class TestReceiver(object): + """Simple Proxy class so the consumer has methods to call. + + Uses static methods because we aren't actually storing any state. + + """ + + @staticmethod + def echo(context, value): + """Simply returns whatever value is sent in.""" + LOG.debug(_("Received %s"), value) + return value + + @staticmethod + def context(context, value): + """Returns dictionary version of context.""" + LOG.debug(_("Received %s"), context) + return context.to_dict() + + @staticmethod + def echo_three_times(context, value): + context.reply(value) + context.reply(value + 1) + context.reply(value + 2) + + @staticmethod + def echo_three_times_yield(context, value): + yield value + yield value + 1 + yield value + 2 + + @staticmethod + def fail(context, value): + """Raises an exception with the value sent in.""" + raise Exception(value) diff --git a/nova/utils.py b/nova/utils.py index 8784a227d..ad31f88bd 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -809,3 +809,14 @@ class Bootstrapper(object): for key in FLAGS: value = FLAGS.get(key, None) logging.audit(_("%(key)s : %(value)s" % locals())) + + +def load_module(name): + mod = __import__(name) + + components = name.split('.') + + for comp in components[1:]: + mod = getattr(mod, comp) + + return mod From 2deb35055838c9f97dbc580dbd2f8f2f20caeb28 Mon Sep 17 00:00:00 2001 From: Zed Shaw Date: Tue, 26 Jul 2011 16:43:04 -0700 Subject: [PATCH 20/91] Add myself to authors. --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 1a07946bd..e82070002 100644 --- a/Authors +++ b/Authors @@ -105,3 +105,4 @@ Yoshiaki Tamura Youcef Laribi Yuriy Taraday Zhixue Wu +Zed Shaw From 9c5746bc2072bf80b05424a8de6c676772c9e5e9 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Wed, 27 Jul 2011 12:50:52 -0400 Subject: [PATCH 21/91] pep8 --- run_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 8f2b51757..e39ecd315 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -84,7 +84,9 @@ function run_pep8 { srcfiles+=" `find tools/*`" srcfiles+=" nova setup.py plugins/xenserver/xenapi/etc/xapi.d/plugins/glance" # Just run PEP8 in current environment - ${wrapper} pep8 --repeat --show-pep8 --show-source \ + #${wrapper} pep8 --repeat --show-pep8 --show-source \ + #--exclude=vcsversion.py ${srcfiles} + pep8 --repeat --show-pep8 --show-source \ --exclude=vcsversion.py ${srcfiles} } From 92bd2222944fced8333c3905012a2b784583684d Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Wed, 27 Jul 2011 17:16:46 +0000 Subject: [PATCH 22/91] Add context argument a lot more places and make unit tests work --- nova/tests/test_libvirt.py | 6 +++--- nova/tests/test_xenapi.py | 16 +++++++++------- nova/tests/xenapi/stubs.py | 4 ++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index ad0931a89..c4af38426 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -365,7 +365,7 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = connection.LibvirtConnection(False) - conn.snapshot(instance_ref, recv_meta['id']) + conn.snapshot(self.context, instance_ref, recv_meta['id']) snapshot = image_service.show(context, recv_meta['id']) self.assertEquals(snapshot['properties']['image_state'], 'available') @@ -405,7 +405,7 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = connection.LibvirtConnection(False) - conn.snapshot(instance_ref, recv_meta['id']) + conn.snapshot(self.context, instance_ref, recv_meta['id']) snapshot = image_service.show(context, recv_meta['id']) self.assertEquals(snapshot['properties']['image_state'], 'available') @@ -775,7 +775,7 @@ class LibvirtConnTestCase(test.TestCase): network_info = [(network, mapping)] try: - conn.spawn(instance, network_info) + conn.spawn(context.get_admin_context(), instance, network_info) except Exception, e: count = (0 <= str(e.message).find('Unexpected method call')) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 199a8bc52..fd8416424 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -227,7 +227,7 @@ class XenAPIVMTestCase(test.TestCase): 'mac': 'DE:AD:BE:EF:00:00', 'rxtx_cap': 3})] instance = db.instance_create(self.context, values) - self.conn.spawn(instance, network_info) + self.conn.spawn(self.context, instance, network_info) gt1 = eventlet.spawn(_do_build, 1, self.project.id, self.user.id) gt2 = eventlet.spawn(_do_build, 2, self.project.id, self.user.id) @@ -257,14 +257,15 @@ class XenAPIVMTestCase(test.TestCase): instance = self._create_instance() name = "MySnapshot" - self.assertRaises(exception.Error, self.conn.snapshot, instance, name) + self.assertRaises(exception.Error, self.conn.snapshot, + self.context, instance, name) def test_instance_snapshot(self): stubs.stubout_instance_snapshot(self.stubs) instance = self._create_instance() name = "MySnapshot" - template_vm_ref = self.conn.snapshot(instance, name) + template_vm_ref = self.conn.snapshot(self.context, instance, name) def ensure_vm_was_torn_down(): vm_labels = [] @@ -422,7 +423,7 @@ class XenAPIVMTestCase(test.TestCase): 'label': 'fake', 'mac': 'DE:AD:BE:EF:00:00', 'rxtx_cap': 3})] - self.conn.spawn(instance, network_info) + self.conn.spawn(self.context, instance, network_info) self.create_vm_record(self.conn, os_type, instance_id) self.check_vm_record(self.conn, check_injection) self.assertTrue(instance.os_type) @@ -691,7 +692,7 @@ class XenAPIVMTestCase(test.TestCase): 'label': 'fake', 'mac': 'DE:AD:BE:EF:00:00', 'rxtx_cap': 3})] - self.conn.spawn(instance, network_info) + self.conn.spawn(self.context, instance, network_info) return instance @@ -802,8 +803,9 @@ class XenAPIMigrateInstance(test.TestCase): 'label': 'fake', 'mac': 'DE:AD:BE:EF:00:00', 'rxtx_cap': 3})] - conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), - network_info) + conn.finish_resize(self.context, instance, + dict(base_copy='hurr', cow='durr'), + network_info) class XenAPIDetermineDiskImageTestCase(test.TestCase): diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 66c79d465..3a142081c 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -28,8 +28,8 @@ from nova import utils def stubout_instance_snapshot(stubs): @classmethod - def fake_fetch_image(cls, session, instance_id, image, user, project, - type): + def fake_fetch_image(cls, context, session, instance_id, image, user, + project, type): from nova.virt.xenapi.fake import create_vdi name_label = "instance-%s" % instance_id #TODO: create fake SR record From 97d5af0cc09a4892b6310939c923030122b1ad38 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Wed, 27 Jul 2011 17:56:12 +0000 Subject: [PATCH 23/91] Fix context argument in a test; add TODOs --- nova/tests/test_libvirt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index c4af38426..d4f8f00d8 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -775,7 +775,7 @@ class LibvirtConnTestCase(test.TestCase): network_info = [(network, mapping)] try: - conn.spawn(context.get_admin_context(), instance, network_info) + conn.spawn(self.context, instance, network_info) except Exception, e: count = (0 <= str(e.message).find('Unexpected method call')) From b32824cb2483aaf39110e9f1024d4c12e24c5288 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 27 Jul 2011 13:00:37 -0500 Subject: [PATCH 24/91] Add a flag to set the default file mode of logs. --- nova/flags.py | 2 +- nova/log.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index 49355b436..fa6d8860a 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -343,7 +343,7 @@ DEFINE_string('lock_path', os.path.join(os.path.dirname(__file__), '../'), 'Directory for lock files') DEFINE_string('logdir', None, 'output to a per-service log file in named ' 'directory') - +DEFINE_integer('logfile_mode', 0644, 'Default file mode of the logs.') DEFINE_string('sqlite_db', 'nova.sqlite', 'file name for sqlite') DEFINE_string('sql_connection', 'sqlite:///$state_path/$sqlite_db', diff --git a/nova/log.py b/nova/log.py index f8c0ba68d..133ee45f8 100644 --- a/nova/log.py +++ b/nova/log.py @@ -257,6 +257,7 @@ class NovaRootLogger(NovaLogger): self.filelog = WatchedFileHandler(logpath) self.addHandler(self.filelog) self.logpath = logpath + os.chmod(self.logpath, FLAGS.logfile_mode) else: self.removeHandler(self.filelog) self.addHandler(self.streamlog) From e67700551b8c320a3c8e8087caa0daa37aeb6e06 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Wed, 27 Jul 2011 17:54:35 -0400 Subject: [PATCH 25/91] put run_tests.sh back to how it was --- run_tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index e39ecd315..8f2b51757 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -84,9 +84,7 @@ function run_pep8 { srcfiles+=" `find tools/*`" srcfiles+=" nova setup.py plugins/xenserver/xenapi/etc/xapi.d/plugins/glance" # Just run PEP8 in current environment - #${wrapper} pep8 --repeat --show-pep8 --show-source \ - #--exclude=vcsversion.py ${srcfiles} - pep8 --repeat --show-pep8 --show-source \ + ${wrapper} pep8 --repeat --show-pep8 --show-source \ --exclude=vcsversion.py ${srcfiles} } From 859a23481296074a297f7a28ad23592a0ebe4990 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Wed, 27 Jul 2011 20:32:46 -0400 Subject: [PATCH 26/91] fix call to nonexistant method to_global_ipv6. Add myself to authors file --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 1a07946bd..c31858365 100644 --- a/Authors +++ b/Authors @@ -67,6 +67,7 @@ Lvov Maxim Mark Washenberger Masanori Itoh Matt Dietz +Matthew Hooker Michael Gundlach Mike Scherbakov Mohammed Naser From 3b3bddaa39f268294ebfb7571ee4fffb21f0898e Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Wed, 27 Jul 2011 20:48:26 -0400 Subject: [PATCH 27/91] remove unused assignment which causes undeclared name error --- nova/tests/scheduler/test_scheduler.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py index daea826fd..18fbef5ca 100644 --- a/nova/tests/scheduler/test_scheduler.py +++ b/nova/tests/scheduler/test_scheduler.py @@ -485,11 +485,6 @@ class SimpleDriverTestCase(test.TestCase): self.assertEqual(host, 'host2') volume1.delete_volume(self.context, volume_id1) db.volume_destroy(self.context, volume_id2) - dic = {'service_id': s_ref['id'], - 'vcpus': 16, 'memory_mb': 32, 'local_gb': 100, - 'vcpus_used': 16, 'memory_mb_used': 12, 'local_gb_used': 10, - 'hypervisor_type': 'qemu', 'hypervisor_version': 12003, - 'cpu_info': ''} def test_doesnt_report_disabled_hosts_as_up(self): """Ensures driver doesn't find hosts before they are enabled""" From 8401a5a5ad030f607fdc9f8d69dad04f2c8d1d93 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Wed, 27 Jul 2011 20:55:10 -0400 Subject: [PATCH 28/91] fix undeclared name error --- nova/scheduler/zone_aware_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index c429fdfcc..d99d7214c 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -81,7 +81,7 @@ class ZoneAwareScheduler(driver.Scheduler): decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key) try: json_entry = decryptor(blob) - return json.dumps(entry) + return json.dumps(json_entry) except M2Crypto.EVP.EVPError: pass return None From 1fc3e1635b56c36a7749e20317ef32403f8d539f Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Wed, 27 Jul 2011 20:56:12 -0400 Subject: [PATCH 29/91] fix undeclared name error --- nova/scheduler/least_cost.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py index 6f5eb66fd..8c400d476 100644 --- a/nova/scheduler/least_cost.py +++ b/nova/scheduler/least_cost.py @@ -28,6 +28,7 @@ from nova import flags from nova import log as logging from nova.scheduler import zone_aware_scheduler from nova import utils +from nova import exception LOG = logging.getLogger('nova.scheduler.least_cost') From 5c425aed64009b35915aba21ccfbeaafcd275e2a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 28 Jul 2011 01:31:09 +0000 Subject: [PATCH 30/91] fix tests broken in the merge --- nova/tests/test_db_api.py | 34 +++++++++++++--------------------- nova/tests/test_vmwareapi.py | 2 +- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/nova/tests/test_db_api.py b/nova/tests/test_db_api.py index 107fd03e3..54448f9d6 100644 --- a/nova/tests/test_db_api.py +++ b/nova/tests/test_db_api.py @@ -22,7 +22,6 @@ from nova import test from nova import context from nova import db from nova import flags -from nova.auth import manager FLAGS = flags.FLAGS @@ -45,42 +44,35 @@ def _setup_networking(instance_id, ip='1.2.3.4', flo_addr='1.2.1.2'): db.fixed_ip_create(ctxt, fixed_ip) fix_ref = db.fixed_ip_get_by_address(ctxt, ip) db.floating_ip_create(ctxt, {'address': flo_addr, - 'fixed_ip_id': fix_ref.id}) + 'fixed_ip_id': fix_ref['id']}) class DbApiTestCase(test.TestCase): def setUp(self): super(DbApiTestCase, self).setUp() - self.manager = manager.AuthManager() - self.user = self.manager.create_user('admin', 'admin', 'admin', True) - self.project = self.manager.create_project('proj', 'admin', 'proj') - self.context = context.RequestContext(user=self.user, - project=self.project) - - def tearDown(self): - self.manager.delete_project(self.project) - self.manager.delete_user(self.user) - super(DbApiTestCase, self).tearDown() + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) def test_instance_get_project_vpn(self): - result = db.fixed_ip_get_all(self.context) values = {'instance_type_id': FLAGS.default_instance_type, 'image_ref': FLAGS.vpn_image_id, - 'project_id': self.project.id + 'project_id': self.project_id } instance = db.instance_create(self.context, values) - result = db.instance_get_project_vpn(self.context, self.project.id) - self.assertEqual(instance.id, result.id) + result = db.instance_get_project_vpn(self.context.elevated(), + self.project_id) + self.assertEqual(instance['id'], result['id']) def test_instance_get_project_vpn_joins(self): - result = db.fixed_ip_get_all(self.context) values = {'instance_type_id': FLAGS.default_instance_type, 'image_ref': FLAGS.vpn_image_id, - 'project_id': self.project.id + 'project_id': self.project_id } instance = db.instance_create(self.context, values) - _setup_networking(instance.id) - result = db.instance_get_project_vpn(self.context, self.project.id) - self.assertEqual(instance.id, result.id) + _setup_networking(instance['id']) + result = db.instance_get_project_vpn(self.context.elevated(), + self.project_id) + self.assertEqual(instance['id'], result['id']) self.assertEqual(result['fixed_ips'][0]['floating_ips'][0].address, '1.2.1.2') diff --git a/nova/tests/test_vmwareapi.py b/nova/tests/test_vmwareapi.py index be5246fdf..3d87d67ad 100644 --- a/nova/tests/test_vmwareapi.py +++ b/nova/tests/test_vmwareapi.py @@ -19,6 +19,7 @@ Test suite for VMWareAPI. """ +from nova import context from nova import db from nova import flags from nova import test @@ -42,7 +43,6 @@ class VMWareAPIVMTestCase(test.TestCase): self.flags(vmwareapi_host_ip='test_url', vmwareapi_host_username='test_username', vmwareapi_host_password='test_pass') - self.manager = manager.AuthManager() self.user_id = 'fake' self.project_id = 'fake' self.context = context.RequestContext(self.user_id, self.project_id) From 312a0cbe590cde2736c5612acfd81f6e18fede2c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 28 Jul 2011 08:34:27 -0700 Subject: [PATCH 31/91] make payload json serializable --- nova/notifier/api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 98969fd3e..45105d00f 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -80,6 +80,12 @@ def notify(publisher_id, event_type, priority, payload): if priority not in log_levels: raise BadPriorityException( _('%s not in valid priorities' % priority)) + + # Ensure everything is JSON serializable. + for k, v in payload.iteritems(): + if not isinstance(v, (basestring, int, long, float)): + payload[k] = str(v) + driver = utils.import_object(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), publisher_id=publisher_id, From 473efd9aa430b5df67d10b9ec7963c16a34a7285 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 28 Jul 2011 08:44:01 -0700 Subject: [PATCH 32/91] unicode instead of str() --- nova/notifier/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 45105d00f..8eea2a032 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -84,7 +84,7 @@ def notify(publisher_id, event_type, priority, payload): # Ensure everything is JSON serializable. for k, v in payload.iteritems(): if not isinstance(v, (basestring, int, long, float)): - payload[k] = str(v) + payload[k] = unicode(v) driver = utils.import_object(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), From 68c5f14fe2d4beaec298852538daf15b534d38ba Mon Sep 17 00:00:00 2001 From: Jason Koelker Date: Thu, 28 Jul 2011 11:01:38 -0500 Subject: [PATCH 33/91] trunk infected with non-pep8 code --- nova/tests/test_db_api.py | 4 ++-- nova/tests/test_libvirt.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_db_api.py b/nova/tests/test_db_api.py index 107fd03e3..5560b489b 100644 --- a/nova/tests/test_db_api.py +++ b/nova/tests/test_db_api.py @@ -66,7 +66,7 @@ class DbApiTestCase(test.TestCase): result = db.fixed_ip_get_all(self.context) values = {'instance_type_id': FLAGS.default_instance_type, 'image_ref': FLAGS.vpn_image_id, - 'project_id': self.project.id + 'project_id': self.project.id, } instance = db.instance_create(self.context, values) result = db.instance_get_project_vpn(self.context, self.project.id) @@ -76,7 +76,7 @@ class DbApiTestCase(test.TestCase): result = db.fixed_ip_get_all(self.context) values = {'instance_type_id': FLAGS.default_instance_type, 'image_ref': FLAGS.vpn_image_id, - 'project_id': self.project.id + 'project_id': self.project.id, } instance = db.instance_create(self.context, values) _setup_networking(instance.id) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index bab4d200b..f367688cc 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -239,7 +239,7 @@ class LibvirtConnTestCase(test.TestCase): 'mac_address': 'fake', 'ip_address': 'fake', 'dhcp_server': 'fake', - 'extra_params': 'fake' + 'extra_params': 'fake', } # Creating mocks From 33a97f01cdbe6c02ba61d0440f9e9b53fe37cde0 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 28 Jul 2011 12:52:47 -0500 Subject: [PATCH 34/91] Catch DBError for duplicate projects. --- nova/auth/dbdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/dbdriver.py b/nova/auth/dbdriver.py index a429b7812..0158c4e23 100644 --- a/nova/auth/dbdriver.py +++ b/nova/auth/dbdriver.py @@ -127,7 +127,7 @@ class DbDriver(object): try: project = db.project_create(context.get_admin_context(), values) - except exception.Duplicate: + except (exception.Duplicate, exception.DBError): raise exception.ProjectExists(project=name) for member in members: From 6a60d5b11e860cc326349d8f68b7595cbdddf10d Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 28 Jul 2011 13:50:06 -0500 Subject: [PATCH 35/91] Removed unused Duplicate catch. --- nova/auth/dbdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/auth/dbdriver.py b/nova/auth/dbdriver.py index 0158c4e23..c6d81ee04 100644 --- a/nova/auth/dbdriver.py +++ b/nova/auth/dbdriver.py @@ -127,7 +127,7 @@ class DbDriver(object): try: project = db.project_create(context.get_admin_context(), values) - except (exception.Duplicate, exception.DBError): + except exception.DBError: raise exception.ProjectExists(project=name) for member in members: From 74b481396d106e65976ed3964da92737beb8282a Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 28 Jul 2011 15:09:02 -0400 Subject: [PATCH 36/91] Some tests for resolved pylint errors. --- .../scheduler/test_zone_aware_scheduler.py | 20 +++++++++++++++++++ nova/tests/test_xenapi.py | 15 ++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py index d74b71fb6..14f8191f2 100644 --- a/nova/tests/scheduler/test_zone_aware_scheduler.py +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -16,6 +16,9 @@ Tests For Zone Aware Scheduler. """ +import json +import mox + import nova.db from nova import exception @@ -327,3 +330,20 @@ class ZoneAwareSchedulerTestCase(test.TestCase): sched._provision_resource_from_blob(None, request_spec, 1, request_spec, {}) self.assertTrue(was_called) + + def test_decrypt_blob(self): + """Test that the decrypt method works.""" + + fixture = FakeZoneAwareScheduler() + test_data = {"foo": "bar"} + + crypto = self.mox.CreateMockAnything() + crypto.decryptor(mox.IgnoreArg()).AndReturn(lambda blob: blob) + """ + def _decryptor(i): + return lambda blob: blob + """ + self.stubs.Set(zone_aware_scheduler, 'crypto', + crypto) + + self.assertEqual(fixture._decrypt_blob(test_data), json.dumps(test_data)) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 8b3b5fa28..781a218b6 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -807,6 +807,21 @@ class XenAPIMigrateInstance(test.TestCase): conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), network_info) +class XenAPIImageTypeTestCase(test.TestCase): + """Test ImageType class.""" + + def test_to_string(self): + """Can convert from type id to type string.""" + self.assertEquals( + vm_utils.ImageType.to_string(vm_utils.ImageType.KERNEL), + vm_utils.ImageType.KERNEL_STR) + + def test_from_string(self): + """Can convert from string to type id.""" + self.assertEquals( + vm_utils.ImageType.to_string(vm_utils.ImageType.KERNEL_STR), + vm_utils.ImageType.KERNEL) + class XenAPIDetermineDiskImageTestCase(test.TestCase): """Unit tests for code that detects the ImageType.""" From 961bf55f846a6ea49788707dbe548c7c11d51571 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 28 Jul 2011 15:46:20 -0400 Subject: [PATCH 37/91] Fix tests for checking pylint errors. --- nova/tests/scheduler/test_zone_aware_scheduler.py | 12 +++++------- nova/tests/test_xenapi.py | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py index 14f8191f2..3781c567e 100644 --- a/nova/tests/scheduler/test_zone_aware_scheduler.py +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -337,13 +337,11 @@ class ZoneAwareSchedulerTestCase(test.TestCase): fixture = FakeZoneAwareScheduler() test_data = {"foo": "bar"} - crypto = self.mox.CreateMockAnything() - crypto.decryptor(mox.IgnoreArg()).AndReturn(lambda blob: blob) - """ - def _decryptor(i): - return lambda blob: blob - """ + class StubDecryptor(object): + def decryptor(self, key): + return lambda blob: blob + self.stubs.Set(zone_aware_scheduler, 'crypto', - crypto) + StubDecryptor()) self.assertEqual(fixture._decrypt_blob(test_data), json.dumps(test_data)) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 781a218b6..e87622451 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -819,7 +819,7 @@ class XenAPIImageTypeTestCase(test.TestCase): def test_from_string(self): """Can convert from string to type id.""" self.assertEquals( - vm_utils.ImageType.to_string(vm_utils.ImageType.KERNEL_STR), + vm_utils.ImageType.from_string(vm_utils.ImageType.KERNEL_STR), vm_utils.ImageType.KERNEL) From 0414c40ffcf49c5f927ca96da6426593fb761937 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 28 Jul 2011 17:41:13 -0400 Subject: [PATCH 38/91] fix pep8 errors --- nova/tests/scheduler/test_zone_aware_scheduler.py | 5 +++-- nova/tests/test_xenapi.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py index 3781c567e..043d0d5c8 100644 --- a/nova/tests/scheduler/test_zone_aware_scheduler.py +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -343,5 +343,6 @@ class ZoneAwareSchedulerTestCase(test.TestCase): self.stubs.Set(zone_aware_scheduler, 'crypto', StubDecryptor()) - - self.assertEqual(fixture._decrypt_blob(test_data), json.dumps(test_data)) + + self.assertEqual(fixture._decrypt_blob(test_data), + json.dumps(test_data)) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index e87622451..df95f2c60 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -807,6 +807,7 @@ class XenAPIMigrateInstance(test.TestCase): conn.finish_resize(instance, dict(base_copy='hurr', cow='durr'), network_info) + class XenAPIImageTypeTestCase(test.TestCase): """Test ImageType class.""" From 39db7c08b325dea80e2719cdad916dfcb761fe18 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 29 Jul 2011 07:05:27 -0700 Subject: [PATCH 39/91] added instance support to to_primitive and tests --- nova/notifier/api.py | 4 +--- nova/utils.py | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 8eea2a032..70264efa8 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -82,9 +82,7 @@ def notify(publisher_id, event_type, priority, payload): _('%s not in valid priorities' % priority)) # Ensure everything is JSON serializable. - for k, v in payload.iteritems(): - if not isinstance(v, (basestring, int, long, float)): - payload[k] = unicode(v) + payload = utils.to_primitive(payload) driver = utils.import_object(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), diff --git a/nova/utils.py b/nova/utils.py index 8784a227d..f54bf72ed 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -521,6 +521,9 @@ def to_primitive(value): return to_primitive(dict(value.iteritems())) elif hasattr(value, '__iter__'): return to_primitive(list(value)) + elif hasattr(value, '__dict__'): + # Class member variables not supported. + return to_primitive(value.__dict__) else: return value From 1ac1fb04898a1ad551f62fb4d98b6e993e006a3a Mon Sep 17 00:00:00 2001 From: William Wolf Date: Fri, 29 Jul 2011 10:08:20 -0400 Subject: [PATCH 40/91] fix more spacing issues, and removed self link from versions template data --- run_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 8f2b51757..e39ecd315 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -84,7 +84,9 @@ function run_pep8 { srcfiles+=" `find tools/*`" srcfiles+=" nova setup.py plugins/xenserver/xenapi/etc/xapi.d/plugins/glance" # Just run PEP8 in current environment - ${wrapper} pep8 --repeat --show-pep8 --show-source \ + #${wrapper} pep8 --repeat --show-pep8 --show-source \ + #--exclude=vcsversion.py ${srcfiles} + pep8 --repeat --show-pep8 --show-source \ --exclude=vcsversion.py ${srcfiles} } From 5c9f55be0e8752a2fa2417ae707da1d2feb09b5c Mon Sep 17 00:00:00 2001 From: William Wolf Date: Fri, 29 Jul 2011 10:54:20 -0400 Subject: [PATCH 41/91] fix run_tests.sh --- run_tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index e39ecd315..8f2b51757 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -84,9 +84,7 @@ function run_pep8 { srcfiles+=" `find tools/*`" srcfiles+=" nova setup.py plugins/xenserver/xenapi/etc/xapi.d/plugins/glance" # Just run PEP8 in current environment - #${wrapper} pep8 --repeat --show-pep8 --show-source \ - #--exclude=vcsversion.py ${srcfiles} - pep8 --repeat --show-pep8 --show-source \ + ${wrapper} pep8 --repeat --show-pep8 --show-source \ --exclude=vcsversion.py ${srcfiles} } From c79f23f780435025cd13f368635b68ec02e9c352 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 12:21:46 -0400 Subject: [PATCH 42/91] updating HACKING --- HACKING | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/HACKING b/HACKING index 2f364c894..2dcb76591 100644 --- a/HACKING +++ b/HACKING @@ -10,13 +10,14 @@ Imports - thou shalt not import objects, only modules - thou shalt not import more than one module per line - thou shalt not make relative imports +- thou shalt order your imports by the full module path - thou shalt organize your imports according to the following template :: # vim: tabstop=4 shiftwidth=4 softtabstop=4 - {{stdlib imports in human alphabetical order}} + {{stdlib imports in human alphabetical order by module name}} \n - {{nova imports in human alphabetical order}} + {{nova imports in human alphabetical order by module name}} \n \n {{begin your code}} @@ -42,11 +43,12 @@ Human Alphabetical Order Examples import time import unittest - from nova import flags - from nova import test + import nova.api.ec2 + from nova.api import openstack from nova.auth import users - from nova.endpoint import api + import nova.flags from nova.endpoint import cloud + from nova import test Docstrings ---------- @@ -70,6 +72,61 @@ Docstrings :param foo: the foo parameter :param bar: the bar parameter - :returns: description of the return value + :returns: return_type -- description of the return value + :raises: AttributeError, KeyError """ + +Dictionaries/Lists +------------------ + If a dictionary (dict) or list object is longer than 80 characters, its + items should be split with newlines. Embedded iterables should have their + items indented. Additionally, the last item in the dictionary should have + a trailing comma. This increases readability and simplifies future diffs. + + Example: + + my_dictionary = { + "image": { + "name": "Just a Snapshot", + "size": 2749573, + "properties": { + "user_id": 12, + "arch": "x86_64", + }, + "things": [ + "thing_one", + "thing_two", + ], + "status": "ACTIVE", + }, + } + +Method Signatures +----------------- + Calls to methods 80 characters or longer should format each argument with + newlines. This is mainly for readability. + + unnecessarily_long_function_name('string one', + 'string two', + kwarg1=constants.ACTIVE, + kwarg2=['a', 'b', 'c']) + + + Rather than constructing parameters inline, it is better to break things up: + + list_of_strings = [ + 'what_a_long_string', + 'not as long', + ] + + dict_of_numbers = { + 'one': 1, + 'two': 2, + 'twenty four': 24, + } + + object_one.call_a_method('string three', + 'string four', + kwarg1=list_of_strings, + kwarg2=dict_of_numbers) From f2e7e27e559336279b5b539b83ec7186d7108c3f Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 12:44:11 -0400 Subject: [PATCH 43/91] expanding --- HACKING | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/HACKING b/HACKING index 2dcb76591..ee919e205 100644 --- a/HACKING +++ b/HACKING @@ -50,6 +50,7 @@ Human Alphabetical Order Examples from nova.endpoint import cloud from nova import test + Docstrings ---------- """A one line docstring looks like this and ends in a period.""" @@ -77,6 +78,7 @@ Docstrings """ + Dictionaries/Lists ------------------ If a dictionary (dict) or list object is longer than 80 characters, its @@ -101,9 +103,38 @@ Dictionaries/Lists "status": "ACTIVE", }, } + + Only use the dict constructor for casting. Do not use it to create a new + dictionary. -Method Signatures ------------------ + Example (BAD): + + my_dictionary = dict(key1='param1', key2='param2', key3=['a', 'b']) + + +Defining Methods +---------------- + Method signatures longer than 80 characters are very unreadable. If you + encounter this problem, first you should determine if your method is + too big. Otherwise, you should compress your keyword arguments with a + '**kwargs' parameter. You should use the 'kwargs' in your method as a + dictionary to retrieve the necessary keyword arguments. + + Example (BAD): + + def my_method(argument_one, argument_two, kwarg_one='default_one', + kwarg_two='default_two', kwarg_three='default_three'): + + Example (GOOD): + + def my_method(argumet_one, argument_two, **kwargs): + kwarg_one = kwargs.get('kwarg_one', 'default_one') + kwarg_two = kwargs.get('kwarg_one', 'default_one') + kwarg_three = kwargs.get('kwarg_three', 'default_three') + + +Calling Methods +--------------- Calls to methods 80 characters or longer should format each argument with newlines. This is mainly for readability. @@ -130,3 +161,26 @@ Method Signatures 'string four', kwarg1=list_of_strings, kwarg2=dict_of_numbers) + +Internationalization (i18n) Strings +---------------------------- + In order to support multiple languages, we have a mechanism to support + automatic translations of exception and log strings. + + Example: + msg = _("An error occurred") + raise HTTPBadRequest(explanation=msg) + + If you have a variable to place within the string, first internationalize + the template string then do the replacement. + + Example: + msg = _("Missing parameter: %s") % ("flavor",) + LOG.error(msg) + + If you have multiple variables to place in the string, use keyword + parameters. This helps our translators reorder parameters when needed. + + Example: + msg = _("The server with id %(s_id)s has no key %(m_key)s") + LOG.error(msg % (s_id="1234, m_key="imageId")) From e528d9dd7675ebdf76fa57c7a6ecd5e3ec07d45a Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 12:49:48 -0400 Subject: [PATCH 44/91] upgrades --- HACKING | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/HACKING b/HACKING index ee919e205..82ccdb962 100644 --- a/HACKING +++ b/HACKING @@ -5,13 +5,23 @@ Step 1: Read http://www.python.org/dev/peps/pep-0008/ Step 2: Read http://www.python.org/dev/peps/pep-0008/ again Step 3: Read on + +General +------- +- Put two newlines twixt toplevel code (funcs, classes, etc) +- Put one newline twixt methods in classes and anywhere else +- Do not write "except:", use "except Exception:" at the very least +- Include your name with TODOs as in "TODO(termie)" +- Do not name anything the same name as a builtin or reserved word + + Imports ------- -- thou shalt not import objects, only modules -- thou shalt not import more than one module per line -- thou shalt not make relative imports -- thou shalt order your imports by the full module path -- thou shalt organize your imports according to the following template +- Do not import objects, only modules +- Do not import more than one module per line +- Do not make relative imports +- Order your imports by the full module path +- Organize your imports according to the following template :: # vim: tabstop=4 shiftwidth=4 softtabstop=4 @@ -23,16 +33,6 @@ Imports {{begin your code}} -General -------- -- thou shalt put two newlines twixt toplevel code (funcs, classes, etc) -- thou shalt put one newline twixt methods in classes and anywhere else -- thou shalt not write "except:", use "except Exception:" at the very least -- thou shalt include your name with TODOs as in "TODO(termie)" -- thou shalt not name anything the same name as a builtin or reserved word -- thou shalt not violate causality in our time cone, or else - - Human Alphabetical Order Examples --------------------------------- :: From 3a8be11642bf9ae1b2e0960fe6f3d16a7314873b Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 12:54:55 -0400 Subject: [PATCH 45/91] one last change --- HACKING | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/HACKING b/HACKING index 82ccdb962..9d6b43b98 100644 --- a/HACKING +++ b/HACKING @@ -8,11 +8,11 @@ Step 3: Read on General ------- -- Put two newlines twixt toplevel code (funcs, classes, etc) -- Put one newline twixt methods in classes and anywhere else +- Put two newlines between top-level code (funcs, classes, etc) +- Put one newline between methods in classes and anywhere else - Do not write "except:", use "except Exception:" at the very least -- Include your name with TODOs as in "TODO(termie)" -- Do not name anything the same name as a builtin or reserved word +- Include your name with TODOs as in "#TODO(termie)" +- Do not name anything the same name as a built-in or reserved word Imports From 344167d6b4db09b53b75b8400047d8cff321e7e3 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 13:00:09 -0400 Subject: [PATCH 46/91] rewording --- HACKING | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HACKING b/HACKING index 9d6b43b98..cb5f696d1 100644 --- a/HACKING +++ b/HACKING @@ -136,7 +136,7 @@ Defining Methods Calling Methods --------------- Calls to methods 80 characters or longer should format each argument with - newlines. This is mainly for readability. + newlines. This is not a requirement, but a guideline. unnecessarily_long_function_name('string one', 'string two', From d5840ef2f7e992f4093d111cf0e9c39125770ab9 Mon Sep 17 00:00:00 2001 From: Zed Shaw Date: Fri, 29 Jul 2011 10:33:58 -0700 Subject: [PATCH 47/91] Use the util.import_object to import a module. --- nova/rpc.py | 4 ++-- nova/utils.py | 11 ----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 8b0c6df67..f5dd49982 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -17,7 +17,7 @@ # under the License. -from nova.utils import load_module +from nova.utils import import_object from nova.rpc_backends.common import RemoteError, LOG from nova import flags @@ -26,7 +26,7 @@ flags.DEFINE_string('rpc_backend', 'nova.rpc_backends.amqp', "The messaging module to use, defaults to AMQP.") -RPCIMPL = load_module(FLAGS.rpc_backend) +RPCIMPL = import_object(FLAGS.rpc_backend) def create_connection(new=True): diff --git a/nova/utils.py b/nova/utils.py index ad31f88bd..8784a227d 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -809,14 +809,3 @@ class Bootstrapper(object): for key in FLAGS: value = FLAGS.get(key, None) logging.audit(_("%(key)s : %(value)s" % locals())) - - -def load_module(name): - mod = __import__(name) - - components = name.split('.') - - for comp in components[1:]: - mod = getattr(mod, comp) - - return mod From 89f3e30ed4bb1af495bd082cdb804bf9ccb2b07a Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 13:46:24 -0400 Subject: [PATCH 48/91] rewording --- HACKING | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/HACKING b/HACKING index cb5f696d1..4c5193779 100644 --- a/HACKING +++ b/HACKING @@ -116,16 +116,12 @@ Defining Methods ---------------- Method signatures longer than 80 characters are very unreadable. If you encounter this problem, first you should determine if your method is - too big. Otherwise, you should compress your keyword arguments with a - '**kwargs' parameter. You should use the 'kwargs' in your method as a - dictionary to retrieve the necessary keyword arguments. + too big. If not, you can compress your keyword arguments with a + '**kwargs' parameter. You can then use 'kwargs' in your method as a + dictionary to retrieve the necessary keyword arguments. This is just a + guideline, not a requirement. - Example (BAD): - - def my_method(argument_one, argument_two, kwarg_one='default_one', - kwarg_two='default_two', kwarg_three='default_three'): - - Example (GOOD): + Example: def my_method(argumet_one, argument_two, **kwargs): kwarg_one = kwargs.get('kwarg_one', 'default_one') From b1a7fb1739793127bceb49130cb023beaae9d016 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 13:51:49 -0400 Subject: [PATCH 49/91] removing 'Defining Methods' paragraph --- HACKING | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/HACKING b/HACKING index 4c5193779..ea0188141 100644 --- a/HACKING +++ b/HACKING @@ -112,23 +112,6 @@ Dictionaries/Lists my_dictionary = dict(key1='param1', key2='param2', key3=['a', 'b']) -Defining Methods ----------------- - Method signatures longer than 80 characters are very unreadable. If you - encounter this problem, first you should determine if your method is - too big. If not, you can compress your keyword arguments with a - '**kwargs' parameter. You can then use 'kwargs' in your method as a - dictionary to retrieve the necessary keyword arguments. This is just a - guideline, not a requirement. - - Example: - - def my_method(argumet_one, argument_two, **kwargs): - kwarg_one = kwargs.get('kwarg_one', 'default_one') - kwarg_two = kwargs.get('kwarg_one', 'default_one') - kwarg_three = kwargs.get('kwarg_three', 'default_three') - - Calling Methods --------------- Calls to methods 80 characters or longer should format each argument with @@ -158,6 +141,7 @@ Calling Methods kwarg1=list_of_strings, kwarg2=dict_of_numbers) + Internationalization (i18n) Strings ---------------------------- In order to support multiple languages, we have a mechanism to support From bea45301f7bdaae69e9bfdb3d47a40a83ab85652 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Fri, 29 Jul 2011 14:58:49 -0400 Subject: [PATCH 50/91] remove unused import --- nova/tests/scheduler/test_zone_aware_scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py index 043d0d5c8..7833028c3 100644 --- a/nova/tests/scheduler/test_zone_aware_scheduler.py +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -17,7 +17,6 @@ Tests For Zone Aware Scheduler. """ import json -import mox import nova.db From ecba6156ca7ec5eb21a95bdf079a2bf25993f32b Mon Sep 17 00:00:00 2001 From: Zed Shaw Date: Fri, 29 Jul 2011 12:08:59 -0700 Subject: [PATCH 51/91] Reorganize the code to satisfy review comments. --- nova/{rpc.py => rpc/__init__.py} | 4 ++-- nova/{rpc_backends => rpc}/amqp.py | 2 +- nova/{rpc_backends => rpc}/common.py | 0 nova/rpc_backends/__init__.py | 0 nova/tests/test_rpc_amqp.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename nova/{rpc.py => rpc/__init__.py} (95%) rename nova/{rpc_backends => rpc}/amqp.py (99%) rename nova/{rpc_backends => rpc}/common.py (100%) delete mode 100644 nova/rpc_backends/__init__.py diff --git a/nova/rpc.py b/nova/rpc/__init__.py similarity index 95% rename from nova/rpc.py rename to nova/rpc/__init__.py index f5dd49982..bdf7f705b 100644 --- a/nova/rpc.py +++ b/nova/rpc/__init__.py @@ -18,12 +18,12 @@ from nova.utils import import_object -from nova.rpc_backends.common import RemoteError, LOG +from nova.rpc.common import RemoteError, LOG from nova import flags FLAGS = flags.FLAGS flags.DEFINE_string('rpc_backend', - 'nova.rpc_backends.amqp', + 'nova.rpc.amqp', "The messaging module to use, defaults to AMQP.") RPCIMPL = import_object(FLAGS.rpc_backend) diff --git a/nova/rpc_backends/amqp.py b/nova/rpc/amqp.py similarity index 99% rename from nova/rpc_backends/amqp.py rename to nova/rpc/amqp.py index efa178bd2..61555795a 100644 --- a/nova/rpc_backends/amqp.py +++ b/nova/rpc/amqp.py @@ -44,7 +44,7 @@ from nova import fakerabbit from nova import flags from nova import log as logging from nova import utils -from nova.rpc_backends.common import RemoteError, LOG +from nova.rpc.common import RemoteError, LOG FLAGS = flags.FLAGS diff --git a/nova/rpc_backends/common.py b/nova/rpc/common.py similarity index 100% rename from nova/rpc_backends/common.py rename to nova/rpc/common.py diff --git a/nova/rpc_backends/__init__.py b/nova/rpc_backends/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/nova/tests/test_rpc_amqp.py b/nova/tests/test_rpc_amqp.py index e3df2393a..d29f7ae32 100644 --- a/nova/tests/test_rpc_amqp.py +++ b/nova/tests/test_rpc_amqp.py @@ -2,7 +2,7 @@ from nova import context from nova import flags from nova import log as logging from nova import rpc -from nova.rpc_backends import amqp +from nova.rpc import amqp from nova import test From a7589f9fbd2f791d9a6fd04ee482adebcd0636f9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 29 Jul 2011 12:09:17 -0700 Subject: [PATCH 52/91] made the whole instance handling thing optional --- nova/notifier/api.py | 2 +- nova/utils.py | 77 +++++++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 70264efa8..e18f3e280 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -82,7 +82,7 @@ def notify(publisher_id, event_type, priority, payload): _('%s not in valid priorities' % priority)) # Ensure everything is JSON serializable. - payload = utils.to_primitive(payload) + payload = utils.to_primitive(payload, convert_instances=True) driver = utils.import_object(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), diff --git a/nova/utils.py b/nova/utils.py index f54bf72ed..1a04c471c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -504,28 +504,61 @@ def utf8(value): return value -def to_primitive(value): - if type(value) is type([]) or type(value) is type((None,)): - o = [] - for v in value: - o.append(to_primitive(v)) - return o - elif type(value) is type({}): - o = {} - for k, v in value.iteritems(): - o[k] = to_primitive(v) - return o - elif isinstance(value, datetime.datetime): - return str(value) - elif hasattr(value, 'iteritems'): - return to_primitive(dict(value.iteritems())) - elif hasattr(value, '__iter__'): - return to_primitive(list(value)) - elif hasattr(value, '__dict__'): - # Class member variables not supported. - return to_primitive(value.__dict__) - else: - return value +def to_primitive(value, convert_instances=False, level=0): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + if inspect.isclass(value): + return unicode(value) + + if level > 3: + return [] + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + if type(value) is type([]) or type(value) is type((None,)): + o = [] + for v in value: + o.append(to_primitive(v, convert_instances=convert_instances, + level=level)) + return o + elif type(value) is type({}): + o = {} + for k, v in value.iteritems(): + o[k] = to_primitive(v, convert_instances=convert_instances, + level=level) + return o + elif isinstance(value, datetime.datetime): + return str(value) + elif hasattr(value, 'iteritems'): + return to_primitive(dict(value.iteritems()), + convert_instances=convert_instances, + level=level) + elif hasattr(value, '__iter__'): + return to_primitive(list(value), level) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return to_primitive(value.__dict__, + convert_instances=convert_instances, + level=level + 1) + else: + return value + except TypeError, e: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return unicode(value) def dumps(value): From 2a5169ee0b8a5ba6115a694e1b22d91a38280b06 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 15:26:14 -0400 Subject: [PATCH 53/91] adding more on return_type in docstrings --- HACKING | 1 + 1 file changed, 1 insertion(+) diff --git a/HACKING b/HACKING index ea0188141..2f33fd19f 100644 --- a/HACKING +++ b/HACKING @@ -74,6 +74,7 @@ Docstrings :param foo: the foo parameter :param bar: the bar parameter :returns: return_type -- description of the return value + :returns: description of the return value :raises: AttributeError, KeyError """ From a9c485908852cc9cb94e9bb8e35238fd2686f6b0 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 15:32:27 -0400 Subject: [PATCH 54/91] removing dict() comment --- HACKING | 7 ------- 1 file changed, 7 deletions(-) diff --git a/HACKING b/HACKING index 2f33fd19f..9e1b29668 100644 --- a/HACKING +++ b/HACKING @@ -105,13 +105,6 @@ Dictionaries/Lists }, } - Only use the dict constructor for casting. Do not use it to create a new - dictionary. - - Example (BAD): - - my_dictionary = dict(key1='param1', key2='param2', key3=['a', 'b']) - Calling Methods --------------- From a5c9f33f6149c735eeb62b5a7ca1d853a718de65 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 15:42:10 -0400 Subject: [PATCH 55/91] removing extra verbage --- HACKING | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HACKING b/HACKING index 9e1b29668..70b2b09e7 100644 --- a/HACKING +++ b/HACKING @@ -25,9 +25,9 @@ Imports :: # vim: tabstop=4 shiftwidth=4 softtabstop=4 - {{stdlib imports in human alphabetical order by module name}} + {{stdlib imports in human alphabetical order}} \n - {{nova imports in human alphabetical order by module name}} + {{nova imports in human alphabetical order}} \n \n {{begin your code}} From 2219a9f474c572a01814773f62292d926f44c7ca Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 15:43:23 -0400 Subject: [PATCH 56/91] fixing underline --- HACKING | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HACKING b/HACKING index 70b2b09e7..3d79e9e82 100644 --- a/HACKING +++ b/HACKING @@ -137,7 +137,7 @@ Calling Methods Internationalization (i18n) Strings ----------------------------- +----------------------------------- In order to support multiple languages, we have a mechanism to support automatic translations of exception and log strings. From 6cf36e20eee7fba2166e5288d6e1a61aa459bf63 Mon Sep 17 00:00:00 2001 From: Jason Koelker Date: Fri, 29 Jul 2011 14:53:38 -0500 Subject: [PATCH 57/91] require either v4 or v6 --- bin/nova-manage | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 75d74903c..3c50e4fc8 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -662,8 +662,9 @@ class NetworkCommands(object): # check for certain required inputs if not label: raise exception.NetworkNotCreated(req='--label') - if not fixed_range_v4: - raise exception.NetworkNotCreated(req='--fixed_range_v4') + if not (fixed_range_v4 or fixed_range_v6): + req = '--fixed_range_v4 or --fixed_range_v6' + raise exception.NetworkNotCreated(req=req) bridge = bridge or FLAGS.flat_network_bridge if not bridge: From cc9b87d8ad1ba34835cceafa40918951c053e7f1 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Fri, 29 Jul 2011 15:58:41 -0400 Subject: [PATCH 58/91] typo --- HACKING | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HACKING b/HACKING index 3d79e9e82..232a353fd 100644 --- a/HACKING +++ b/HACKING @@ -157,4 +157,4 @@ Internationalization (i18n) Strings Example: msg = _("The server with id %(s_id)s has no key %(m_key)s") - LOG.error(msg % (s_id="1234, m_key="imageId")) + LOG.error(msg % {"s_id": "1234", "m_key": "imageId"}) From 5e37dde2024cc4cc3e5a4ef188d6e85f3e761176 Mon Sep 17 00:00:00 2001 From: Jason Koelker Date: Fri, 29 Jul 2011 15:05:53 -0500 Subject: [PATCH 59/91] stwart the switch to just fixed_range --- bin/nova-manage | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 3c50e4fc8..1c3bd9984 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -632,6 +632,7 @@ class NetworkCommands(object): @args('--label', dest="label", metavar='