merging trunk

This commit is contained in:
Brian Waldon
2011-06-13 12:54:43 -04:00
14 changed files with 398 additions and 89 deletions

View File

@@ -31,6 +31,7 @@ Hisaharu Ishii <ishii.hisaharu@lab.ntt.co.jp>
Hisaki Ohara <hisaki.ohara@intel.com>
Ilya Alekseyev <ialekseev@griddynamics.com>
Isaku Yamahata <yamahata@valinux.co.jp>
Jason Cannavale <jason.cannavale@rackspace.com>
Jason Koelker <jason@koelker.net>
Jay Pipes <jaypipes@gmail.com>
Jesse Andrews <anotherjesse@gmail.com>

View File

@@ -96,6 +96,7 @@ flags.DECLARE('network_size', 'nova.network.manager')
flags.DECLARE('vlan_start', 'nova.network.manager')
flags.DECLARE('vpn_start', 'nova.network.manager')
flags.DECLARE('fixed_range_v6', 'nova.network.manager')
flags.DECLARE('gateway_v6', 'nova.network.manager')
flags.DECLARE('libvirt_type', 'nova.virt.libvirt.connection')
flags.DEFINE_flag(flags.HelpFlag())
flags.DEFINE_flag(flags.HelpshortFlag())
@@ -544,13 +545,10 @@ class FloatingIpCommands(object):
class NetworkCommands(object):
"""Class for managing networks."""
def create(self, fixed_range=None, num_networks=None,
network_size=None, vlan_start=None,
vpn_start=None, fixed_range_v6=None, label='public'):
"""Creates fixed ips for host by range
arguments: fixed_range=FLAG, [num_networks=FLAG],
[network_size=FLAG], [vlan_start=FLAG],
[vpn_start=FLAG], [fixed_range_v6=FLAG]"""
def create(self, fixed_range=None, num_networks=None, network_size=None,
vlan_start=None, vpn_start=None, fixed_range_v6=None,
gateway_v6=None, label='public'):
"""Creates fixed ips for host by range"""
if not fixed_range:
msg = _('Fixed range in the form of 10.0.0.0/8 is '
'required to create networks.')
@@ -566,6 +564,8 @@ class NetworkCommands(object):
vpn_start = FLAGS.vpn_start
if not fixed_range_v6:
fixed_range_v6 = FLAGS.fixed_range_v6
if not gateway_v6:
gateway_v6 = FLAGS.gateway_v6
net_manager = utils.import_object(FLAGS.network_manager)
try:
net_manager.create_networks(context.get_admin_context(),
@@ -575,6 +575,7 @@ class NetworkCommands(object):
vlan_start=int(vlan_start),
vpn_start=int(vpn_start),
cidr_v6=fixed_range_v6,
gateway_v6=gateway_v6,
label=label)
except ValueError, e:
print e

View File

@@ -139,7 +139,7 @@ class LdapDriver(object):
self.__cache = None
return False
def __local_cache(key_fmt):
def __local_cache(key_fmt): # pylint: disable=E0213
"""Wrap function to cache it's result in self.__cache.
Works only with functions with one fixed argument.
"""

View File

@@ -14,4 +14,5 @@ alias ec2-bundle-image="ec2-bundle-image --cert ${EC2_CERT} --privatekey ${EC2_P
alias ec2-upload-bundle="ec2-upload-bundle -a ${EC2_ACCESS_KEY} -s ${EC2_SECRET_KEY} --url ${S3_URL} --ec2cert ${NOVA_CERT}"
export NOVA_API_KEY="%(access)s"
export NOVA_USERNAME="%(user)s"
export NOVA_PROJECT_ID="%(project)s"
export NOVA_URL="%(os)s"

View File

@@ -270,8 +270,10 @@ DEFINE_list('region_list',
DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake')
DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID')
DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key')
DEFINE_integer('glance_port', 9292, 'glance port')
DEFINE_string('glance_host', '$my_ip', 'glance host')
# NOTE(sirp): my_ip interpolation doesn't work within nested structures
DEFINE_list('glance_api_servers',
['127.0.0.1:9292'],
'list of glance api servers available to nova (host:port)')
DEFINE_integer('s3_port', 3333, 's3 port')
DEFINE_string('s3_host', '$my_ip', 's3 host (for infrastructure)')
DEFINE_string('s3_dmz', '$my_ip', 's3 dmz ip (for instances)')
@@ -381,3 +383,5 @@ DEFINE_string('zone_name', 'nova', 'name of this zone')
DEFINE_list('zone_capabilities',
['hypervisor=xenserver;kvm', 'os=linux;windows'],
'Key/Multi-value list representng capabilities of this zone')
DEFINE_string('build_plan_encryption_key', None,
'128bit (hex) encryption key for scheduler build plans.')

View File

@@ -84,7 +84,7 @@ def get_zone_capabilities(context):
def select(context, specs=None):
"""Returns a list of hosts."""
return _call_scheduler('select', context=context,
params={"specs": specs})
params={"request_spec": specs})
def update_service_capabilities(context, service_name, host, capabilities):

View File

@@ -21,16 +21,30 @@ across zones. There are two expansion points to this class for:
"""
import operator
import json
import M2Crypto
import novaclient
from nova import crypto
from nova import db
from nova import exception
from nova import flags
from nova import log as logging
from nova import rpc
from nova.scheduler import api
from nova.scheduler import driver
FLAGS = flags.FLAGS
LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler')
class InvalidBlob(exception.NovaException):
message = _("Ill-formed or incorrectly routed 'blob' data sent "
"to instance create request.")
class ZoneAwareScheduler(driver.Scheduler):
"""Base class for creating Zone Aware Schedulers."""
@@ -38,6 +52,112 @@ class ZoneAwareScheduler(driver.Scheduler):
"""Call novaclient zone method. Broken out for testing."""
return api.call_zone_method(context, method, specs=specs)
def _provision_resource_locally(self, context, item, instance_id, kwargs):
"""Create the requested resource in this Zone."""
host = item['hostname']
kwargs['instance_id'] = instance_id
rpc.cast(context,
db.queue_get_for(context, "compute", host),
{"method": "run_instance",
"args": kwargs})
LOG.debug(_("Provisioning locally via compute node %(host)s")
% locals())
def _decrypt_blob(self, blob):
"""Returns the decrypted blob or None if invalid. Broken out
for testing."""
decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key)
try:
json_entry = decryptor(blob)
return json.dumps(entry)
except M2Crypto.EVP.EVPError:
pass
return None
def _ask_child_zone_to_create_instance(self, context, zone_info,
request_spec, kwargs):
"""Once we have determined that the request should go to one
of our children, we need to fabricate a new POST /servers/
call with the same parameters that were passed into us.
Note that we have to reverse engineer from our args to get back the
image, flavor, ipgroup, etc. since the original call could have
come in from EC2 (which doesn't use these things)."""
instance_type = request_spec['instance_type']
instance_properties = request_spec['instance_properties']
name = instance_properties['display_name']
image_id = instance_properties['image_id']
meta = instance_properties['metadata']
flavor_id = instance_type['flavorid']
files = kwargs['injected_files']
ipgroup = None # Not supported in OS API ... yet
child_zone = zone_info['child_zone']
child_blob = zone_info['child_blob']
zone = db.zone_get(context, child_zone)
url = zone.api_url
LOG.debug(_("Forwarding instance create call to child zone %(url)s")
% locals())
nova = None
try:
nova = novaclient.OpenStack(zone.username, zone.password, url)
nova.authenticate()
except novaclient.exceptions.BadRequest, e:
raise exception.NotAuthorized(_("Bad credentials attempting "
"to talk to zone at %(url)s.") % locals())
nova.servers.create(name, image_id, flavor_id, ipgroup, meta, files,
child_blob)
def _provision_resource_from_blob(self, context, item, instance_id,
request_spec, kwargs):
"""Create the requested resource locally or in a child zone
based on what is stored in the zone blob info.
Attempt to decrypt the blob to see if this request is:
1. valid, and
2. intended for this zone or a child zone.
Note: If we have "blob" that means the request was passed
into us from a parent zone. If we have "child_blob" that
means we gathered the info from one of our children.
It's possible that, when we decrypt the 'blob' field, it
contains "child_blob" data. In which case we forward the
request."""
host_info = None
if "blob" in item:
# Request was passed in from above. Is it for us?
host_info = self._decrypt_blob(item['blob'])
elif "child_blob" in item:
# Our immediate child zone provided this info ...
host_info = item
if not host_info:
raise InvalidBlob()
# Valid data ... is it for us?
if 'child_zone' in host_info and 'child_blob' in host_info:
self._ask_child_zone_to_create_instance(context, host_info,
request_spec, kwargs)
else:
self._provision_resource_locally(context, host_info,
instance_id, kwargs)
def _provision_resource(self, context, item, instance_id, request_spec,
kwargs):
"""Create the requested resource in this Zone or a child zone."""
if "hostname" in item:
self._provision_resource_locally(context, item, instance_id,
kwargs)
return
self._provision_resource_from_blob(context, item, instance_id,
request_spec, kwargs)
def schedule_run_instance(self, context, instance_id, request_spec,
*args, **kwargs):
"""This method is called from nova.compute.api to provision
@@ -51,8 +171,10 @@ class ZoneAwareScheduler(driver.Scheduler):
# TODO(sandy): We'll have to look for richer specs at some point.
if 'blob' in request_spec:
self.provision_resource(context, request_spec, instance_id, kwargs)
blob = request_spec.get('blob')
if blob:
self._provision_resource(context, request_spec, instance_id,
request_spec, kwargs)
return None
# Create build plan and provision ...
@@ -61,28 +183,13 @@ class ZoneAwareScheduler(driver.Scheduler):
raise driver.NoValidHost(_('No hosts were available'))
for item in build_plan:
self.provision_resource(context, item, instance_id, kwargs)
self._provision_resource(context, item, instance_id, request_spec,
kwargs)
# Returning None short-circuits the routing to Compute (since
# we've already done it here)
return None
def provision_resource(self, context, item, instance_id, kwargs):
"""Create the requested resource in this Zone or a child zone."""
if "hostname" in item:
host = item['hostname']
kwargs['instance_id'] = instance_id
rpc.cast(context,
db.queue_get_for(context, "compute", host),
{"method": "run_instance",
"args": kwargs})
LOG.debug(_("Casted to compute %(host)s for run_instance")
% locals())
else:
# TODO(sandy) Provision in child zone ...
LOG.warning(_("Provision to Child Zone not supported (yet)"))
pass
def select(self, context, request_spec, *args, **kwargs):
"""Select returns a list of weights and zone/host information
corresponding to the best hosts to service the request. Any
@@ -124,17 +231,17 @@ class ZoneAwareScheduler(driver.Scheduler):
weighted = self.weigh_hosts(num_instances, request_spec, host_list)
# Next, tack on the best weights from the child zones ...
json_spec = json.dumps(request_spec)
child_results = self._call_zone_method(context, "select",
specs=request_spec)
specs=json_spec)
for child_zone, result in child_results:
for weighting in result:
# Remember the child_zone so we can get back to
# it later if needed. This implicitly builds a zone
# path structure.
host_dict = {
"weight": weighting["weight"],
"child_zone": child_zone,
"child_blob": weighting["blob"]}
host_dict = {"weight": weighting["weight"],
"child_zone": child_zone,
"child_blob": weighting["blob"]}
weighted.append(host_dict)
weighted.sort(key=operator.itemgetter('weight'))

View File

@@ -16,6 +16,7 @@
Tests For Zone Aware Scheduler.
"""
from nova import exception
from nova import test
from nova.scheduler import driver
from nova.scheduler import zone_aware_scheduler
@@ -90,6 +91,41 @@ def fake_empty_call_zone_method(context, method, specs):
return []
# Hmm, I should probably be using mox for this.
was_called = False
def fake_provision_resource(context, item, instance_id, request_spec, kwargs):
global was_called
was_called = True
def fake_ask_child_zone_to_create_instance(context, zone_info,
request_spec, kwargs):
global was_called
was_called = True
def fake_provision_resource_locally(context, item, instance_id, kwargs):
global was_called
was_called = True
def fake_provision_resource_from_blob(context, item, instance_id,
request_spec, kwargs):
global was_called
was_called = True
def fake_decrypt_blob_returns_local_info(blob):
return {'foo': True} # values aren't important.
def fake_decrypt_blob_returns_child_info(blob):
return {'child_zone': True,
'child_blob': True} # values aren't important. Keys are.
def fake_call_zone_method(context, method, specs):
return [
('zone1', [
@@ -149,4 +185,112 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
fake_context = {}
self.assertRaises(driver.NoValidHost, sched.schedule_run_instance,
fake_context, 1,
dict(host_filter=None, instance_type={}))
dict(host_filter=None,
request_spec={'instance_type': {}}))
def test_schedule_do_not_schedule_with_hint(self):
"""
Check the local/child zone routing in the run_instance() call.
If the zone_blob hint was passed in, don't re-schedule.
"""
global was_called
sched = FakeZoneAwareScheduler()
was_called = False
self.stubs.Set(sched, '_provision_resource', fake_provision_resource)
request_spec = {
'instance_properties': {},
'instance_type': {},
'filter_driver': 'nova.scheduler.host_filter.AllHostsFilter',
'blob': "Non-None blob data"
}
result = sched.schedule_run_instance(None, 1, request_spec)
self.assertEquals(None, result)
self.assertTrue(was_called)
def test_provision_resource_local(self):
"""Provision a resource locally or remotely."""
global was_called
sched = FakeZoneAwareScheduler()
was_called = False
self.stubs.Set(sched, '_provision_resource_locally',
fake_provision_resource_locally)
request_spec = {'hostname': "foo"}
sched._provision_resource(None, request_spec, 1, request_spec, {})
self.assertTrue(was_called)
def test_provision_resource_remote(self):
"""Provision a resource locally or remotely."""
global was_called
sched = FakeZoneAwareScheduler()
was_called = False
self.stubs.Set(sched, '_provision_resource_from_blob',
fake_provision_resource_from_blob)
request_spec = {}
sched._provision_resource(None, request_spec, 1, request_spec, {})
self.assertTrue(was_called)
def test_provision_resource_from_blob_empty(self):
"""Provision a resource locally or remotely given no hints."""
global was_called
sched = FakeZoneAwareScheduler()
request_spec = {}
self.assertRaises(zone_aware_scheduler.InvalidBlob,
sched._provision_resource_from_blob,
None, {}, 1, {}, {})
def test_provision_resource_from_blob_with_local_blob(self):
"""
Provision a resource locally or remotely when blob hint passed in.
"""
global was_called
sched = FakeZoneAwareScheduler()
was_called = False
self.stubs.Set(sched, '_decrypt_blob',
fake_decrypt_blob_returns_local_info)
self.stubs.Set(sched, '_provision_resource_locally',
fake_provision_resource_locally)
request_spec = {'blob': "Non-None blob data"}
sched._provision_resource_from_blob(None, request_spec, 1,
request_spec, {})
self.assertTrue(was_called)
def test_provision_resource_from_blob_with_child_blob(self):
"""
Provision a resource locally or remotely when child blob hint
passed in.
"""
global was_called
sched = FakeZoneAwareScheduler()
self.stubs.Set(sched, '_decrypt_blob',
fake_decrypt_blob_returns_child_info)
was_called = False
self.stubs.Set(sched, '_ask_child_zone_to_create_instance',
fake_ask_child_zone_to_create_instance)
request_spec = {'blob': "Non-None blob data"}
sched._provision_resource_from_blob(None, request_spec, 1,
request_spec, {})
self.assertTrue(was_called)
def test_provision_resource_from_blob_with_immediate_child_blob(self):
"""
Provision a resource locally or remotely when blob hint passed in
from an immediate child.
"""
global was_called
sched = FakeZoneAwareScheduler()
was_called = False
self.stubs.Set(sched, '_ask_child_zone_to_create_instance',
fake_ask_child_zone_to_create_instance)
request_spec = {'child_blob': True, 'child_zone': True}
sched._provision_resource_from_blob(None, request_spec, 1,
request_spec, {})
self.assertTrue(was_called)

View File

@@ -115,6 +115,18 @@ class CloudTestCase(test.TestCase):
public_ip=address)
db.floating_ip_destroy(self.context, address)
def test_allocate_address(self):
address = "10.10.10.10"
allocate = self.cloud.allocate_address
db.floating_ip_create(self.context,
{'address': address,
'host': self.network.host})
self.assertEqual(allocate(self.context)['publicIp'], address)
db.floating_ip_destroy(self.context, address)
self.assertRaises(exception.NoMoreFloatingIps,
allocate,
self.context)
def test_associate_disassociate_address(self):
"""Verifies associate runs cleanly without raising an exception"""
address = "10.10.10.10"
@@ -487,6 +499,21 @@ class CloudTestCase(test.TestCase):
self.assertRaises(exception.ApiError, run_instances,
self.context, **kwargs)
def test_run_instances_image_status_active(self):
kwargs = {'image_id': FLAGS.default_image,
'instance_type': FLAGS.default_instance_type,
'max_count': 1}
run_instances = self.cloud.run_instances
def fake_show_stat_active(self, context, id):
return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1,
'type': 'machine'}, 'status': 'active'}
self.stubs.Set(local.LocalImageService, 'show', fake_show_stat_active)
result = run_instances(self.context, **kwargs)
self.assertEqual(len(result['instancesSet']), 1)
def test_terminate_instances(self):
inst1 = db.instance_create(self.context, {'reservation_id': 'a',
'image_ref': 1,

View File

@@ -14,6 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import eventlet
import mox
import os
@@ -125,6 +126,7 @@ class CacheConcurrencyTestCase(test.TestCase):
class LibvirtConnTestCase(test.TestCase):
def setUp(self):
super(LibvirtConnTestCase, self).setUp()
connection._late_load_cheetah()
@@ -207,6 +209,29 @@ class LibvirtConnTestCase(test.TestCase):
self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn')
connection.LibvirtConnection._conn = fake
def fake_lookup(self, instance_name):
class FakeVirtDomain(object):
def snapshotCreateXML(self, *args):
return None
def XMLDesc(self, *args):
return """
<domain type='kvm'>
<devices>
<disk type='file'>
<source file='filename'/>
</disk>
</devices>
</domain>
"""
return FakeVirtDomain()
def fake_execute(self, *args):
open(args[-1], "a").close()
def create_service(self, **kwargs):
service_ref = {'host': kwargs.get('host', 'dummy'),
'binary': 'nova-compute',
@@ -283,38 +308,11 @@ class LibvirtConnTestCase(test.TestCase):
self._check_xml_and_container(instance_data)
def test_snapshot(self):
if not self.lazy_load_library_exists():
return
FLAGS.image_service = 'nova.image.fake.FakeImageService'
# Only file-based instance storages are supported at the moment
test_xml = """
<domain type='kvm'>
<devices>
<disk type='file'>
<source file='filename'/>
</disk>
</devices>
</domain>
"""
class FakeVirtDomain(object):
def __init__(self):
pass
def snapshotCreateXML(self, *args):
return None
def XMLDesc(self, *args):
return test_xml
def fake_lookup(instance_name):
if instance_name == instance_ref.name:
return FakeVirtDomain()
def fake_execute(*args):
# Touch filename to pass 'with open(out_path)'
open(args[-1], "a").close()
# Start test
image_service = utils.import_object(FLAGS.image_service)
@@ -330,9 +328,49 @@ class LibvirtConnTestCase(test.TestCase):
recv_meta = image_service.create(context, sent_meta)
self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn')
connection.LibvirtConnection._conn.lookupByName = fake_lookup
connection.LibvirtConnection._conn.lookupByName = self.fake_lookup
self.mox.StubOutWithMock(connection.utils, 'execute')
connection.utils.execute = fake_execute
connection.utils.execute = self.fake_execute
self.mox.ReplayAll()
conn = connection.LibvirtConnection(False)
conn.snapshot(instance_ref, recv_meta['id'])
snapshot = image_service.show(context, recv_meta['id'])
self.assertEquals(snapshot['properties']['image_state'], 'available')
self.assertEquals(snapshot['status'], 'active')
self.assertEquals(snapshot['name'], snapshot_name)
def test_snapshot_no_image_architecture(self):
if not self.lazy_load_library_exists():
return
FLAGS.image_service = 'nova.image.fake.FakeImageService'
# Start test
image_service = utils.import_object(FLAGS.image_service)
# Assign image_ref = 2 from nova/images/fakes for testing different
# base image
test_instance = copy.deepcopy(self.test_instance)
test_instance["image_ref"] = "2"
# Assuming that base image already exists in image_service
instance_ref = db.instance_create(self.context, test_instance)
properties = {'instance_id': instance_ref['id'],
'user_id': str(self.context.user_id)}
snapshot_name = 'test-snap'
sent_meta = {'name': snapshot_name, 'is_public': False,
'status': 'creating', 'properties': properties}
# Create new image. It will be updated in snapshot method
# To work with it from snapshot, the single image_service is needed
recv_meta = image_service.create(context, sent_meta)
self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn')
connection.LibvirtConnection._conn.lookupByName = self.fake_lookup
self.mox.StubOutWithMock(connection.utils, 'execute')
connection.utils.execute = self.fake_execute
self.mox.ReplayAll()

View File

@@ -69,7 +69,7 @@ class VMWareAPIVMTestCase(test.TestCase):
'instance_type': 'm1.large',
'mac_address': 'aa:bb:cc:dd:ee:ff',
}
self.instance = db.instance_create(values)
self.instance = db.instance_create(None, values)
def _create_vm(self):
"""Create and spawn the VM."""

View File

@@ -52,7 +52,7 @@ def stub_out_db_instance_api(stubs):
else:
raise NotImplementedError()
def fake_instance_create(values):
def fake_instance_create(context, values):
"""Stubs out the db.instance_create method."""
type_data = INSTANCE_TYPES[values['instance_type']]

View File

@@ -42,20 +42,6 @@ def stubout_instance_snapshot(stubs):
stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image)
def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref,
original_parent_uuid):
from nova.virt.xenapi.fake import create_vdi
name_label = "instance-%s" % instance_id
#TODO: create fake SR record
sr_ref = "fakesr"
vdi_ref = create_vdi(name_label=name_label, read_only=False,
sr_ref=sr_ref, sharable=False)
vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref)
vdi_uuid = vdi_rec['uuid']
return vdi_uuid
stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image)
def fake_parse_xmlrpc_value(val):
return val
@@ -251,10 +237,10 @@ class FakeSessionForMigrationTests(fake.SessionBase):
def __init__(self, uri):
super(FakeSessionForMigrationTests, self).__init__(uri)
def VDI_get_by_uuid(*args):
def VDI_get_by_uuid(self, *args):
return 'hurr'
def VDI_resize_online(*args):
def VDI_resize_online(self, *args):
pass
def VM_start(self, _1, ref, _2, _3):

View File

@@ -78,7 +78,7 @@ def WrapTwistedOptions(wrapped):
self._absorbParameters()
self._absorbHandlers()
super(TwistedOptionsToFlags, self).__init__()
wrapped.__init__(self)
def _absorbFlags(self):
twistd_flags = []
@@ -163,12 +163,12 @@ def WrapTwistedOptions(wrapped):
def parseArgs(self, *args):
# TODO(termie): figure out a decent way of dealing with args
#return
super(TwistedOptionsToFlags, self).parseArgs(*args)
wrapped.parseArgs(self, *args)
def postOptions(self):
self._doHandlers()
super(TwistedOptionsToFlags, self).postOptions()
wrapped.postOptions(self)
def __getitem__(self, key):
key = key.replace('-', '_')