From 005a4e645f8e913c673c6ba07e7b0c8c54f33e1c Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Tue, 21 Dec 2010 14:17:29 -0600 Subject: [PATCH 01/84] Refactored duplicate rpc.cast() calls in nova/compute/api.py. Cleaned up some formatting issues. --- nova/compute/api.py | 47 ++++++++++----------------- nova/compute/manager.py | 32 ++++++++++++++++++ nova/tests/compute_unittest.py | 7 ++++ nova/virt/fake.py | 12 +++++++ nova/virt/xenapi/vmops.py | 15 +++++++-- nova/virt/xenapi_conn.py | 59 ++++++++++++++++++---------------- 6 files changed, 111 insertions(+), 61 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index c740814da5af..ae7c84bc5f47 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -206,8 +206,7 @@ class ComputeAPI(base.Base): def delete_instance(self, context, instance_id): logging.debug("Going to try and terminate %d" % instance_id) try: - instance = self.db.instance_get_by_internal_id(context, - instance_id) + instance = self.get_instance(context, instance_id) except exception.NotFound as e: logging.warning("Instance %d was not found during terminate", instance_id) @@ -271,50 +270,38 @@ class ComputeAPI(base.Base): def get_instance(self, context, instance_id): return self.db.instance_get_by_internal_id(context, instance_id) - def reboot(self, context, instance_id): - """Reboot the given instance.""" - instance = self.db.instance_get_by_internal_id(context, instance_id) + def _cast_compute_message(method, context, instance_id): + """Generic handler for RPC calls.""" + instance = self.get_instance(context, instance_id) host = instance['host'] rpc.cast(context, self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "reboot_instance", + {"method": method, "args": {"instance_id": instance['id']}}) + def reboot(self, context, instance_id): + """Reboot the given instance.""" + self._cast_compute_message("reboot_instance", context, instance_id) + def pause(self, context, instance_id): """Pause the given instance.""" - instance = self.db.instance_get_by_internal_id(context, instance_id) - host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "pause_instance", - "args": {"instance_id": instance['id']}}) + self._cast_compute_message("pause_instance", context, instance_id) def unpause(self, context, instance_id): """Unpause the given instance.""" - instance = self.db.instance_get_by_internal_id(context, instance_id) - host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "unpause_instance", - "args": {"instance_id": instance['id']}}) + self._cast_compute_message("unpause_instance", context, instance_id) def rescue(self, context, instance_id): """Rescue the given instance.""" - instance = self.db.instance_get_by_internal_id(context, instance_id) - host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "rescue_instance", - "args": {"instance_id": instance['id']}}) + self._cast_compute_message("rescue_instance", context, instance_id) def unrescue(self, context, instance_id): """Unrescue the given instance.""" - instance = self.db.instance_get_by_internal_id(context, instance_id) - host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "unrescue_instance", - "args": {"instance_id": instance['id']}}) + self._cast_compute_message("unrescue_instance", context, instance_id) + + def reset_root_password(self, context, instance_id): + """Reset the root/admin pw for the given instance.""" + self._cast_compute_message("reset_root_password", context, instance_id) def _get_network_topic(self, context): """Retrieves the network host for a project""" diff --git a/nova/compute/manager.py b/nova/compute/manager.py index a84af6bb91ad..cb64cd39d05e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -156,6 +156,38 @@ class ComputeManager(manager.Manager): self.driver.reboot(instance_ref) self._update_state(context, instance_id) + + + + # WORKING CODE + @exception.wrap_exception + def reset_root_password(self, context, instance_id): + """Reset the root/admin password for an instance on this server.""" + context = context.elevated() + instance_ref = self.db.instance_get(context, instance_id) + self._update_state(context, instance_id) + + if instance_ref['state'] != power_state.RUNNING: + logging.warn('trying to reset the password on a non-running ' + 'instance: %s (state: %s excepted: %s)', + instance_ref['internal_id'], + instance_ref['state'], + power_state.RUNNING) + + logging.debug('instance %s: resetting root password', + instance_ref['name']) + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'resetting_password') + #TODO: (dabo) not sure how we will implement this yet. + self.driver.reset_root_password(instance_ref) + self._update_state(context, instance_id) + + + + + @exception.wrap_exception def rescue_instance(self, context, instance_id): """Rescue an instance on this server.""" diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 187ca31deb28..16e577c56ec9 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -142,6 +142,13 @@ class ComputeTestCase(test.TestCase): self.compute.reboot_instance(self.context, instance_id) self.compute.terminate_instance(self.context, instance_id) + def test_reset_root_password(self): + """Ensure instance can have its root password reset""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.reset_root_password(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) + def test_console_output(self): """Make sure we can get console output from instance""" instance_id = self._create_instance() diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 55c6dcef9e0b..ff3f61838007 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -118,6 +118,18 @@ class FakeConnection(object): """ pass + def reset_root_password(self, instance): + """ + Reset the root password on the specified instance. + + The given parameter is an instance of nova.compute.service.Instance, + and so the instance is being specified as instance.name. + + The work will be done asynchronously. This function returns a + Deferred that allows the caller to detect when it is complete. + """ + pass + def rescue(self, instance): """ Rescue the specified instance. diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index a18eacf07b2f..c5ae52add625 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -34,7 +34,6 @@ class VMOps(object): """ Management class for VM-related tasks """ - def __init__(self, session): global XenAPI if XenAPI is None: @@ -45,8 +44,9 @@ class VMOps(object): def list_instances(self): """ List VM instances """ - return [self._session.get_xenapi().VM.get_name_label(vm) \ - for vm in self._session.get_xenapi().VM.get_all()] + xVM = self._session.get_xenapi().VM + return [xVM.get_name_label(vm) + for vm in xVM.get_all()] def spawn(self, instance): """ Create VM instance """ @@ -89,6 +89,15 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.clean_reboot', vm) self._session.wait_for_task(task) + def reset_root_password(self, instance): + """ Reset the root/admin password on the VM instance """ + instance_name = instance.name + vm = VMHelper.lookup(self._session, instance_name) + if vm is None: + raise Exception('instance not present %s' % instance_name) + task = self._session.call_xenapi('Async.VM.reset_root_password', vm) + self._session.wait_for_task(task) + def destroy(self, instance): """ Destroy VM instance """ vm = VMHelper.lookup(self._session, instance.name) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 21ed2cd659cd..199c0b862add 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -19,15 +19,15 @@ A connection to XenServer or Xen Cloud Platform. The concurrency model for this class is as follows: -All XenAPI calls are on a thread (using t.i.t.deferToThread, via the decorator -deferredToThread). They are remote calls, and so may hang for the usual -reasons. They should not be allowed to block the reactor thread. +All XenAPI calls are on a green thread (using eventlet's "tpool" +thread pool). They are remote calls, and so may hang for the usual +reasons. All long-running XenAPI calls (VM.start, VM.reboot, etc) are called async -(using XenAPI.VM.async_start etc). These return a task, which can then be -polled for completion. Polling is handled using reactor.callLater. +(using XenAPI.VM.async_start etc). These return a task, which can then be +polled for completion. -This combination of techniques means that we don't block the reactor thread at +This combination of techniques means that we don't block the main thread at all, and at the same time we don't hold lots of threads waiting for long-running operations. @@ -75,7 +75,7 @@ flags.DEFINE_string('xenapi_connection_password', flags.DEFINE_float('xenapi_task_poll_interval', 0.5, 'The interval used for polling of remote tasks ' - '(Async.VM.start, etc). Used only if ' + '(Async.VM.start, etc). Used only if ' 'connection_type=xenapi.') XenAPI = None @@ -101,7 +101,7 @@ def get_connection(_): class XenAPIConnection(object): - """ A connection to XenServer or Xen Cloud Platform """ + """A connection to XenServer or Xen Cloud Platform""" def __init__(self, url, user, pw): session = XenAPISession(url, user, pw) @@ -109,31 +109,35 @@ class XenAPIConnection(object): self._volumeops = VolumeOps(session) def list_instances(self): - """ List VM instances """ + """List VM instances""" return self._vmops.list_instances() def spawn(self, instance): - """ Create VM instance """ + """Create VM instance""" self._vmops.spawn(instance) def reboot(self, instance): - """ Reboot VM instance """ + """Reboot VM instance""" self._vmops.reboot(instance) + def reset_root_password(self, instance): + """Reset the root/admin password on the VM instance""" + self._vmops.reset_root_password(instance) + def destroy(self, instance): - """ Destroy VM instance """ + """Destroy VM instance""" self._vmops.destroy(instance) def pause(self, instance, callback): - """ Pause VM instance """ + """Pause VM instance""" self._vmops.pause(instance, callback) def unpause(self, instance, callback): - """ Unpause paused VM instance """ + """Unpause paused VM instance""" self._vmops.unpause(instance, callback) def get_info(self, instance_id): - """ Return data about VM instance """ + """Return data about VM instance""" return self._vmops.get_info(instance_id) def get_diagnostics(self, instance_id): @@ -141,33 +145,33 @@ class XenAPIConnection(object): return self._vmops.get_diagnostics(instance_id) def get_console_output(self, instance): - """ Return snapshot of console """ + """Return snapshot of console""" return self._vmops.get_console_output(instance) def attach_volume(self, instance_name, device_path, mountpoint): - """ Attach volume storage to VM instance """ + """Attach volume storage to VM instance""" return self._volumeops.attach_volume(instance_name, - device_path, - mountpoint) + device_path, + mountpoint) def detach_volume(self, instance_name, mountpoint): - """ Detach volume storage to VM instance """ + """Detach volume storage to VM instance""" return self._volumeops.detach_volume(instance_name, mountpoint) class XenAPISession(object): - """ The session to invoke XenAPI SDK calls """ + """The session to invoke XenAPI SDK calls""" def __init__(self, url, user, pw): self._session = XenAPI.Session(url) self._session.login_with_password(user, pw) def get_xenapi(self): - """ Return the xenapi object """ + """Return the xenapi object""" return self._session.xenapi def get_xenapi_host(self): - """ Return the xenapi host """ + """Return the xenapi host""" return self._session.xenapi.session.get_this_host(self._session.handle) def call_xenapi(self, method, *args): @@ -184,9 +188,8 @@ class XenAPISession(object): self.get_xenapi_host(), plugin, fn, args) def wait_for_task(self, task): - """Return a Deferred that will give the result of the given task. + """Return the result of the given task. The task is polled until it completes.""" - done = event.Event() loop = utils.LoopingCall(self._poll_task, task, done) loop.start(FLAGS.xenapi_task_poll_interval, now=True) @@ -195,7 +198,7 @@ class XenAPISession(object): return rv def _poll_task(self, task, done): - """Poll the given XenAPI task, and fire the given Deferred if we + """Poll the given XenAPI task, and fire the given action if we get a result.""" try: #logging.debug('Polling task %s...', task) @@ -218,7 +221,7 @@ class XenAPISession(object): def _unwrap_plugin_exceptions(func, *args, **kwargs): - """ Parse exception details """ + """Parse exception details""" try: return func(*args, **kwargs) except XenAPI.Failure, exc: @@ -240,7 +243,7 @@ def _unwrap_plugin_exceptions(func, *args, **kwargs): def _parse_xmlrpc_value(val): - """Parse the given value as if it were an XML-RPC value. This is + """Parse the given value as if it were an XML-RPC value. This is sometimes used as the format for the task.result field.""" if not val: return val From 269ab03f74ea94a586f6af5b7d61847443522ba1 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Wed, 22 Dec 2010 11:20:30 -0600 Subject: [PATCH 02/84] committing so that I can merge trunk changes --- nova/compute/manager.py | 10 +--------- nova/virt/xenapi/vmops.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index cb64cd39d05e..86f8d9216352 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -156,10 +156,6 @@ class ComputeManager(manager.Manager): self.driver.reboot(instance_ref) self._update_state(context, instance_id) - - - - # WORKING CODE @exception.wrap_exception def reset_root_password(self, context, instance_id): """Reset the root/admin password for an instance on this server.""" @@ -180,14 +176,10 @@ class ComputeManager(manager.Manager): instance_id, power_state.NOSTATE, 'resetting_password') - #TODO: (dabo) not sure how we will implement this yet. + #### TODO: (dabo) not sure how we will implement this yet. self.driver.reset_root_password(instance_ref) self._update_state(context, instance_id) - - - - @exception.wrap_exception def rescue_instance(self, context, instance_id): """Rescue an instance on this server.""" diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index c5ae52add625..9dca55e2686e 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -43,13 +43,13 @@ class VMOps(object): VMHelper.late_import() def list_instances(self): - """ List VM instances """ + """List VM instances""" xVM = self._session.get_xenapi().VM return [xVM.get_name_label(vm) for vm in xVM.get_all()] def spawn(self, instance): - """ Create VM instance """ + """Create VM instance""" vm = VMHelper.lookup(self._session, instance.name) if vm is not None: raise Exception('Attempted to create non-unique name %s' % @@ -81,7 +81,7 @@ class VMOps(object): vm_ref) def reboot(self, instance): - """ Reboot VM instance """ + """Reboot VM instance""" instance_name = instance.name vm = VMHelper.lookup(self._session, instance_name) if vm is None: @@ -90,16 +90,18 @@ class VMOps(object): self._session.wait_for_task(task) def reset_root_password(self, instance): - """ Reset the root/admin password on the VM instance """ + """Reset the root/admin password on the VM instance""" instance_name = instance.name vm = VMHelper.lookup(self._session, instance_name) if vm is None: raise Exception('instance not present %s' % instance_name) - task = self._session.call_xenapi('Async.VM.reset_root_password', vm) + #### TODO: (dabo) Need to figure out the correct command to + #### write to the xenstore. + task = self._session.call_xenapi('VM.get xenstore data', vm) self._session.wait_for_task(task) def destroy(self, instance): - """ Destroy VM instance """ + """Destroy VM instance""" vm = VMHelper.lookup(self._session, instance.name) if vm is None: # Don't complain, just return. This lets us clean up instances @@ -136,7 +138,7 @@ class VMOps(object): callback(ret) def pause(self, instance, callback): - """ Pause VM instance """ + """Pause VM instance""" instance_name = instance.name vm = VMHelper.lookup(self._session, instance_name) if vm is None: @@ -145,7 +147,7 @@ class VMOps(object): self._wait_with_callback(task, callback) def unpause(self, instance, callback): - """ Unpause VM instance """ + """Unpause VM instance""" instance_name = instance.name vm = VMHelper.lookup(self._session, instance_name) if vm is None: @@ -154,7 +156,7 @@ class VMOps(object): self._wait_with_callback(task, callback) def get_info(self, instance_id): - """ Return data about VM instance """ + """Return data about VM instance""" vm = VMHelper.lookup_blocking(self._session, instance_id) if vm is None: raise Exception('instance not present %s' % instance_id) @@ -170,6 +172,6 @@ class VMOps(object): return VMHelper.compile_diagnostics(self._session, rec) def get_console_output(self, instance): - """ Return snapshot of console """ + """Return snapshot of console""" # TODO: implement this to fix pylint! return 'FAKE CONSOLE OUTPUT of instance' From 6c8fe1963e6d64ba76698dbbaeb7ef9f63cfda95 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Wed, 22 Dec 2010 17:33:21 -0600 Subject: [PATCH 03/84] Got basic xenstore operations working --- nova/virt/xenapi/vmops.py | 63 ++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 1c76d2ccca9c..e36770cfc200 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -92,16 +92,63 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.clean_reboot', vm) self._session.wait_for_task(instance.id, task) - def reset_root_password(self, instance): - """Reset the root/admin password on the VM instance""" - instance_name = instance.name - vm = VMHelper.lookup(self._session, instance_name) + def _get_vm_opaque_ref(self, instance_or_vm): + try: + instance_name = instance_or_vm.name + vm = VMHelper.lookup(self._session, instance_name) + except AttributeError: + # A vm opaque ref was passed + vm = instance_or_vm if vm is None: raise Exception('instance not present %s' % instance_name) - #### TODO: (dabo) Need to figure out the correct command to - #### write to the xenstore. - task = self._session.call_xenapi('VM.get_xenstore_data', vm) - self._session.wait_for_task(task) + return vm + + def remove_from_xenstore(self, instance_or_vm, keys): + vm = self._get_vm_opaque_ref(instance_or_vm) + for key in keys: + self._session._session.xenapi_request('VM.remove_from_xenstore_data', + (vm, key)) + + def read_from_xenstore(self, instance_or_vm, keys=None): + """Returns the xenstore data for the specified VM instance as + a dict. Accepts an optional list of keys; if the list of keys is + passed, the returned dict is filtered to only return the values + for those keys. + """ + vm = self._get_vm_opaque_ref(instance_or_vm) + ret = self._session._session.xenapi_request('VM.get_xenstore_data', (vm, )) + if keys: + allkeys = set(ret.keys()) + badkeys = allkeys.difference(keys) + for k in badkeys: + ret.pop(k) + return ret + + def add_to_xenstore(self, instance_or_vm, mapping): + """Takes a dict and adds it to the xenstore record for + the given vm instance. Existing data is preserved, but any + existing values for the mapping's keys are overwritten. + """ + vm = self._get_vm_opaque_ref(instance_or_vm) + current_data = self.read_from_xenstore(vm) + current_data.update(mapping) + self.write_to_xenstore(vm, current_data) + + def write_to_xenstore(self, instance_or_vm, mapping): + """Takes a dict and writes it to the xenstore record for + the given vm instance. Any existing data is overwritten. + """ + vm = self._get_vm_opaque_ref(instance_or_vm) + self._session._session.xenapi_request('VM.set_xenstore_data', + (vm, mapping)) + + def reset_root_password(self, instance): + """Reset the root/admin password on the VM instance""" + self.add_to_xenstore(instance, {"reset_root_password": "requested"}) + self.add_to_xenstore(instance, {"TEST": "OMG!"}) + import time + self.add_to_xenstore(instance, {"timestamp": time.ctime()}) + def destroy(self, instance): """Destroy VM instance""" From 4ff2da231d485598232d9aacc41538950005ac34 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 22 Dec 2010 16:43:47 -0800 Subject: [PATCH 04/84] Basic Easy API functionality --- nova/api/easy.py | 163 ++++++++++++++++++++++++++++++++++++ nova/tests/easy_unittest.py | 85 +++++++++++++++++++ nova/wsgi.py | 31 +++++-- run_tests.py | 1 + 4 files changed, 274 insertions(+), 6 deletions(-) create mode 100644 nova/api/easy.py create mode 100644 nova/tests/easy_unittest.py diff --git a/nova/api/easy.py b/nova/api/easy.py new file mode 100644 index 000000000000..a284e9685c4d --- /dev/null +++ b/nova/api/easy.py @@ -0,0 +1,163 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Public HTTP interface that allows services to self-register. + +The general flow of a request is: + - Request is parsed into WSGI bits. + - Some middleware checks authentication. + - Routing takes place based on the URL to find a controller. + (/controller/method) + - Parameters are parsed from the request and passed to a method on the + controller as keyword arguments. + - Optionally json_body is decoded to provide all the parameters. + - Actual work is done and a result is returned. + - That result is turned into json and returned. + +""" + +import json +import urllib + +import routes +import webob + +from nova import context +from nova import flags +from nova import wsgi + +# prxy compute_api in amazon tests + + +EASY_ROUTES = {} + + +def register_service(path, handle): + EASY_ROUTES[path] = handle + + +class DelegatedAuthMiddleware(wsgi.Middleware): + def process_request(self, request): + os_user = request.headers['X-OpenStack-User'] + os_project = request.headers['X-OpenStack-Project'] + context_ref = context.RequestContext(user=os_user, project=os_project) + request.environ['openstack.context'] = context_ref + + +class JsonParamsMiddleware(wsgi.Middleware): + def process_request(self, request): + if 'json' not in request.params: + return + + params_json = request.params['json'] + params_parsed = json.loads(params_json) + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v + + request.environ['openstack.params'] = params + + +class ReqParamsMiddleware(wsgi.Middleware): + def process_request(self, request): + params_parsed = request.params + params = {} + for k, v in params_parsed.iteritems(): + if k in ('self', 'context'): + continue + if k.startswith('_'): + continue + params[k] = v + + request.environ['openstack.params'] = params + + +class SundayMorning(wsgi.Router): + def __init__(self, mapper=None): + if mapper is None: + mapper = routes.Mapper() + + self._load_registered_routes(mapper) + super(SundayMorning, self).__init__(mapper=mapper) + + def _load_registered_routes(self, mapper): + for route in EASY_ROUTES: + mapper.connect('/%s/{action}' % route, + controller=ServiceWrapper(EASY_ROUTES[route])) + + +class ServiceWrapper(wsgi.Controller): + def __init__(self, service_handle): + self.service_handle = service_handle + + @webob.dec.wsgify + def __call__(self, req): + arg_dict = req.environ['wsgiorg.routing_args'][1] + action = arg_dict['action'] + del arg_dict['action'] + + context = req.environ['openstack.context'] + # allow middleware up the stack to override the params + params = {} + if 'openstack.params' in req.environ: + params = req.environ['openstack.params'] + + # TODO(termie): do some basic normalization on methods + method = getattr(self.service_handle, action) + + result = method(context, **params) + if type(result) is dict: + return self._serialize(result, req) + else: + return result + + +class Proxy(object): + """Pretend an Easy API endpoint is an object.""" + def __init__(self, app, prefix=None): + self.app = app + self.prefix = prefix + + def __do_request(self, path, context, **kwargs): + req = webob.Request.blank(path) + req.method = 'POST' + req.body = urllib.urlencode({'json': json.dumps(kwargs)}) + req.environ['openstack.context'] = context + resp = req.get_response(self.app) + try: + return json.loads(resp.body) + except Exception: + return resp.body + + def __getattr__(self, key): + if self.prefix is None: + return self.__class__(self.app, key) + + def _wrapper(context, **kwargs): + return self.__do_request('/%s/%s' % (self.prefix, key), + context, + **kwargs) + _wrapper.func_name = key + return _wrapper + + + diff --git a/nova/tests/easy_unittest.py b/nova/tests/easy_unittest.py new file mode 100644 index 000000000000..ed223831f65c --- /dev/null +++ b/nova/tests/easy_unittest.py @@ -0,0 +1,85 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for Easy API.""" + +import json +import logging + +import webob + +from nova import context +from nova import exception +from nova import test +from nova import utils +from nova.api import easy + + +class FakeService(object): + def echo(self, context, data): + return {'data': data} + + def context(self, context): + return {'user': context.user_id, + 'project': context.project_id} + + +class EasyTestCase(test.TestCase): + def setUp(self): + super(EasyTestCase, self).setUp() + easy.register_service('fake', FakeService()) + self.router = easy.ReqParamsMiddleware( + easy.JsonParamsMiddleware( + easy.SundayMorning())) + self.auth_router = easy.DelegatedAuthMiddleware(self.router) + self.context = context.RequestContext('user1', 'proj1') + + def tearDown(self): + easy.EASY_ROUTES = {} + + def test_delegated_auth(self): + req = webob.Request.blank('/fake/context') + req.headers['X-OpenStack-User'] = 'user1' + req.headers['X-OpenStack-Project'] = 'proj1' + resp = req.get_response(self.auth_router) + data = json.loads(resp.body) + self.assertEqual(data['user'], 'user1') + self.assertEqual(data['project'], 'proj1') + + def test_json_params(self): + req = webob.Request.blank('/fake/echo') + req.environ['openstack.context'] = self.context + req.method = 'POST' + req.body = 'json=%s' % json.dumps({'data': 'foo'}) + resp = req.get_response(self.router) + resp_parsed = json.loads(resp.body) + self.assertEqual(resp_parsed['data'], 'foo') + + def test_req_params(self): + req = webob.Request.blank('/fake/echo') + req.environ['openstack.context'] = self.context + req.method = 'POST' + req.body = 'data=foo' + resp = req.get_response(self.router) + resp_parsed = json.loads(resp.body) + self.assertEqual(resp_parsed['data'], 'foo') + + def test_proxy(self): + proxy = easy.Proxy(self.router) + rv = proxy.fake.echo(self.context, data='baz') + self.assertEqual(rv['data'], 'baz') diff --git a/nova/wsgi.py b/nova/wsgi.py index c7ee9ed146e2..2ad27dd7ef34 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -104,20 +104,39 @@ class Application(object): class Middleware(Application): - """ - Base WSGI middleware wrapper. These classes require an application to be + """Base WSGI middleware. + + Modelled after Django's middleware this class allows you to + These classes require an application to be initialized that will be called next. By default the middleware will simply call its wrapped app, or you can override __call__ to customize its behavior. """ - def __init__(self, application): # pylint: disable-msg=W0231 + def __init__(self, application): self.application = application + def process_request(self, req): + """Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + @webob.dec.wsgify - def __call__(self, req): # pylint: disable-msg=W0221 - """Override to implement middleware behavior.""" - return self.application + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + return self.process_response(response) class Debug(Middleware): diff --git a/run_tests.py b/run_tests.py index 6a4b7f1abaa2..d3cf8f696408 100644 --- a/run_tests.py +++ b/run_tests.py @@ -59,6 +59,7 @@ from nova.tests.api_unittest import * from nova.tests.auth_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * +from nova.tests.easy_unittest import * from nova.tests.flags_unittest import * from nova.tests.misc_unittest import * from nova.tests.network_unittest import * From 43e9f8727af618fc3e50308cba95b27c67ee83c5 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 22 Dec 2010 16:52:16 -0800 Subject: [PATCH 05/84] remove some yields that snuck in --- nova/tests/cloud_unittest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 70d2c44daaec..1398a9862ed3 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -140,15 +140,15 @@ class CloudTestCase(test.TestCase): kwargs = {'image_id': image_id, 'instance_type': instance_type, 'max_count': max_count} - rv = yield self.cloud.run_instances(self.context, **kwargs) + rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] - output = yield self.cloud.get_console_output(context=self.context, + output = self.cloud.get_console_output(context=self.context, instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) - rv = yield self.cloud.terminate_instances(self.context, [instance_id]) + rv = self.cloud.terminate_instances(self.context, [instance_id]) def test_key_generation(self): result = self._create_key('test') @@ -186,7 +186,7 @@ class CloudTestCase(test.TestCase): kwargs = {'image_id': image_id, 'instance_type': instance_type, 'max_count': max_count} - rv = yield self.cloud.run_instances(self.context, **kwargs) + rv = self.cloud.run_instances(self.context, **kwargs) # TODO: check for proper response instance_id = rv['reservationSet'][0].keys()[0] instance = rv['reservationSet'][0][instance_id][0] @@ -209,7 +209,7 @@ class CloudTestCase(test.TestCase): for instance in reservations[reservations.keys()[0]]: instance_id = instance['instance_id'] logging.debug("Terminating instance %s" % instance_id) - rv = yield self.compute.terminate_instance(instance_id) + rv = self.compute.terminate_instance(instance_id) def test_instance_update_state(self): def instance(num): From 70d254c626e925f6de8408f0ca70f3de28a7307a Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 22 Dec 2010 17:53:42 -0800 Subject: [PATCH 06/84] added tests to ensure the easy api works as a backend for Compute API --- nova/api/easy.py | 10 +++++----- nova/api/ec2/cloud.py | 34 ++++++++++++++++++++-------------- nova/compute/api.py | 19 +++++++++++++++---- nova/tests/cloud_unittest.py | 2 ++ nova/tests/compute_unittest.py | 9 ++++++--- nova/tests/easy_unittest.py | 19 ++++++++++++++++++- nova/utils.py | 31 +++++++++++++++++++++++++++++++ nova/wsgi.py | 6 +++--- 8 files changed, 100 insertions(+), 30 deletions(-) diff --git a/nova/api/easy.py b/nova/api/easy.py index a284e9685c4d..9ef4877393a6 100644 --- a/nova/api/easy.py +++ b/nova/api/easy.py @@ -31,7 +31,6 @@ The general flow of a request is: """ -import json import urllib import routes @@ -39,6 +38,7 @@ import webob from nova import context from nova import flags +from nova import utils from nova import wsgi # prxy compute_api in amazon tests @@ -65,7 +65,7 @@ class JsonParamsMiddleware(wsgi.Middleware): return params_json = request.params['json'] - params_parsed = json.loads(params_json) + params_parsed = utils.loads(params_json) params = {} for k, v in params_parsed.iteritems(): if k in ('self', 'context'): @@ -125,7 +125,7 @@ class ServiceWrapper(wsgi.Controller): method = getattr(self.service_handle, action) result = method(context, **params) - if type(result) is dict: + if type(result) is dict or type(result) is list: return self._serialize(result, req) else: return result @@ -140,11 +140,11 @@ class Proxy(object): def __do_request(self, path, context, **kwargs): req = webob.Request.blank(path) req.method = 'POST' - req.body = urllib.urlencode({'json': json.dumps(kwargs)}) + req.body = urllib.urlencode({'json': utils.dumps(kwargs)}) req.environ['openstack.context'] = context resp = req.get_response(self.app) try: - return json.loads(resp.body) + return utils.loads(resp.body) except Exception: return resp.body diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index e09261f00c3f..61c6c0da1f78 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -118,7 +118,8 @@ class CloudController(object): def _get_mpi_data(self, context, project_id): result = {} - for instance in self.compute_api.get_instances(context, project_id): + for instance in self.compute_api.get_instances(context, + project_id=project_id): if instance['fixed_ip']: line = '%s slots=%d' % (instance['fixed_ip']['address'], instance['vcpus']) @@ -442,7 +443,8 @@ class CloudController(object): # instance_id is passed in as a list of instances ec2_id = instance_id[0] internal_id = ec2_id_to_internal_id(ec2_id) - instance_ref = self.compute_api.get_instance(context, internal_id) + instance_ref = self.compute_api.get_instance(context, + instance_id=internal_id) output = rpc.call(context, '%s.%s' % (FLAGS.compute_topic, instance_ref['host']), @@ -541,7 +543,8 @@ class CloudController(object): if volume_ref['attach_status'] == "attached": raise exception.ApiError(_("Volume is already attached")) internal_id = ec2_id_to_internal_id(instance_id) - instance_ref = self.compute_api.get_instance(context, internal_id) + instance_ref = self.compute_api.get_instance(context, + instance_id=internal_id) host = instance_ref['host'] rpc.cast(context, db.queue_get_for(context, FLAGS.compute_topic, host), @@ -722,14 +725,15 @@ class CloudController(object): def associate_address(self, context, instance_id, public_ip, **kwargs): internal_id = ec2_id_to_internal_id(instance_id) - instance_ref = self.compute_api.get_instance(context, internal_id) + instance_ref = self.compute_api.get_instance(context, + instance_id=internal_id) fixed_address = db.instance_get_fixed_address(context, instance_ref['id']) floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) # NOTE(vish): Perhaps we should just pass this on to compute and # let compute communicate with network. - network_topic = self.compute_api.get_network_topic(context, - internal_id) + network_topic = self.compute_api.get_network_topic( + context, instance_id=internal_id) rpc.cast(context, network_topic, {"method": "associate_floating_ip", @@ -754,8 +758,9 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) instances = self.compute_api.create_instances(context, - instance_types.get_by_type(kwargs.get('instance_type', None)), - kwargs['image_id'], + instance_type=instance_types.get_by_type( + kwargs.get('instance_type', None)), + image_id=kwargs['image_id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id', None), @@ -765,7 +770,7 @@ class CloudController(object): key_name=kwargs.get('key_name'), user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), - generate_hostname=internal_id_to_ec2_id) + hostname_format='ec2') return self._format_run_instances(context, instances[0]['reservation_id']) @@ -775,26 +780,26 @@ class CloudController(object): logging.debug("Going to start terminating instances") for ec2_id in instance_id: internal_id = ec2_id_to_internal_id(ec2_id) - self.compute_api.delete_instance(context, internal_id) + self.compute_api.delete_instance(context, instance_id=internal_id) return True def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" for ec2_id in instance_id: internal_id = ec2_id_to_internal_id(ec2_id) - self.compute_api.reboot(context, internal_id) + self.compute_api.reboot(context, instance_id=internal_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" internal_id = ec2_id_to_internal_id(instance_id) - self.compute_api.rescue(context, internal_id) + self.compute_api.rescue(context, instance_id=internal_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" internal_id = ec2_id_to_internal_id(instance_id) - self.compute_api.unrescue(context, internal_id) + self.compute_api.unrescue(context, instance_id=internal_id) return True def update_instance(self, context, ec2_id, **kwargs): @@ -805,7 +810,8 @@ class CloudController(object): changes[field] = kwargs[field] if changes: internal_id = ec2_id_to_internal_id(ec2_id) - inst = self.compute_api.get_instance(context, internal_id) + inst = self.compute_api.get_instance(context, + instance_id=internal_id) db.instance_update(context, inst['id'], kwargs) return True diff --git a/nova/compute/api.py b/nova/compute/api.py index 4953fe559fdb..5f18539a3cbb 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -36,10 +36,19 @@ from nova.db import base FLAGS = flags.FLAGS -def generate_default_hostname(internal_id): +def id_to_default_hostname(internal_id): """Default function to generate a hostname given an instance reference.""" return str(internal_id) +def id_to_ec2_hostname(internal_id): + digits = [] + while internal_id != 0: + internal_id, remainder = divmod(internal_id, 36) + digits.append('0123456789abcdefghijklmnopqrstuvwxyz'[remainder]) + return "i-%s" % ''.join(reversed(digits)) + +HOSTNAME_FORMATTERS = {'default': id_to_default_hostname, + 'ec2': id_to_ec2_hostname} class ComputeAPI(base.Base): """API for interacting with the compute manager.""" @@ -75,7 +84,7 @@ class ComputeAPI(base.Base): display_name='', description='', key_name=None, key_data=None, security_group='default', user_data=None, - generate_hostname=generate_default_hostname): + hostname_format='default'): """Create the number of instances requested if quote and other arguments check out ok.""" @@ -144,6 +153,7 @@ class ComputeAPI(base.Base): elevated = context.elevated() instances = [] + generate_hostname = HOSTNAME_FORMATTERS[hostname_format] logging.debug(_("Going to run %s instances..."), num_instances) for num in range(num_instances): instance = dict(mac_address=utils.generate_mac(), @@ -177,7 +187,7 @@ class ComputeAPI(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id}}) - return instances + return [dict(x.iteritems()) for x in instances] def ensure_default_security_group(self, context): """ Create security group for the security context if it @@ -254,7 +264,8 @@ class ComputeAPI(base.Base): return self.db.instance_get_all(context) def get_instance(self, context, instance_id): - return self.db.instance_get_by_internal_id(context, instance_id) + rv = self.db.instance_get_by_internal_id(context, instance_id) + return dict(rv.iteritems()) def reboot(self, context, instance_id): """Reboot the given instance.""" diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 1398a9862ed3..4439c131c12e 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -22,6 +22,7 @@ import logging from M2Crypto import BIO from M2Crypto import RSA import os +import shutil import tempfile import time @@ -293,6 +294,7 @@ class CloudTestCase(test.TestCase): self.assertEqual('Foo Img', img.metadata['description']) self._fake_set_image_description(self.context, 'ami-testing', '') self.assertEqual('', img.metadata['description']) + shutil.rmtree(pathdir) def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 348bb3351d86..0bebd1c5ea00 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -75,7 +75,7 @@ class ComputeTestCase(test.TestCase): ref = self.compute_api.create_instances(self.context, FLAGS.default_instance_type, None, **instance) try: - self.assertNotEqual(ref[0].display_name, None) + self.assertNotEqual(ref[0]['display_name'], None) finally: db.instance_destroy(self.context, ref[0]['id']) @@ -87,9 +87,12 @@ class ComputeTestCase(test.TestCase): 'project_id': self.project.id} group = db.security_group_create(self.context, values) ref = self.compute_api.create_instances(self.context, - FLAGS.default_instance_type, None, security_group=['default']) + instance_type=FLAGS.default_instance_type, + image_id=None, + security_group=['default']) try: - self.assertEqual(len(ref[0]['security_groups']), 1) + self.assertEqual(len(db.security_group_get_by_instance( + self.context, ref[0]['id'])), 1) finally: db.security_group_destroy(self.context, group['id']) db.instance_destroy(self.context, ref[0]['id']) diff --git a/nova/tests/easy_unittest.py b/nova/tests/easy_unittest.py index ed223831f65c..81990d842e2e 100644 --- a/nova/tests/easy_unittest.py +++ b/nova/tests/easy_unittest.py @@ -28,7 +28,8 @@ from nova import exception from nova import test from nova import utils from nova.api import easy - +from nova.compute import api as compute_api +from nova.tests import cloud_unittest class FakeService(object): def echo(self, context, data): @@ -83,3 +84,19 @@ class EasyTestCase(test.TestCase): proxy = easy.Proxy(self.router) rv = proxy.fake.echo(self.context, data='baz') self.assertEqual(rv['data'], 'baz') + + +class EasyCloudTestCase(cloud_unittest.CloudTestCase): + def setUp(self): + super(EasyCloudTestCase, self).setUp() + compute_handle = compute_api.ComputeAPI(self.cloud.network_manager, + self.cloud.image_service) + easy.register_service('compute', compute_handle) + self.router = easy.JsonParamsMiddleware(easy.SundayMorning()) + proxy = easy.Proxy(self.router) + self.cloud.compute_api = proxy.compute + + def tearDown(self): + super(EasyCloudTestCase, self).tearDown() + easy.EASY_ROUTES = {} + diff --git a/nova/utils.py b/nova/utils.py index b9045a50c976..7a98ffa5a044 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -22,6 +22,7 @@ System-level utilities and helper functions. import datetime import inspect +import json import logging import os import random @@ -361,3 +362,33 @@ def utf8(value): return value.encode("utf-8") assert isinstance(value, str) return value + + +def to_primitive(value): + if type(value) is type([]) or type(value) is type((None,)): + o = [] + for v in value: + o.append(to_primitive(v)) + return o + elif type(value) is type({}): + o = {} + for k, v in value.iteritems(): + o[k] = to_primitive(v) + return o + elif isinstance(value, datetime.datetime): + return str(value) + else: + return value + + +def dumps(value): + try: + return json.dumps(value) + except TypeError: + pass + + return json.dumps(to_primitive(value)) + + +def loads(s): + return json.loads(s) diff --git a/nova/wsgi.py b/nova/wsgi.py index 2ad27dd7ef34..c40f043f929f 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -21,7 +21,6 @@ Utility methods for working with WSGI servers """ -import json import logging import sys from xml.dom import minidom @@ -35,6 +34,7 @@ import webob import webob.dec import webob.exc +from nova import utils logging.getLogger("routes.middleware").addHandler(logging.StreamHandler()) @@ -322,7 +322,7 @@ class Serializer(object): try: is_xml = (datastring[0] == '<') if not is_xml: - return json.loads(datastring) + return utils.loads(datastring) return self._from_xml(datastring) except: return None @@ -355,7 +355,7 @@ class Serializer(object): return result def _to_json(self, data): - return json.dumps(data) + return utils.dumps(data) def _to_xml(self, data): metadata = self.metadata.get('application/xml', {}) From 7c1b3ef521c652ce375390a1ecb04a60d1f100f0 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 22 Dec 2010 18:04:35 -0800 Subject: [PATCH 07/84] remove some notes --- nova/api/easy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/api/easy.py b/nova/api/easy.py index 9ef4877393a6..1be52069fe2b 100644 --- a/nova/api/easy.py +++ b/nova/api/easy.py @@ -41,8 +41,6 @@ from nova import flags from nova import utils from nova import wsgi -# prxy compute_api in amazon tests - EASY_ROUTES = {} From a1b5220879632d093f450413f96668a8f77c0613 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 23 Dec 2010 12:04:19 -0800 Subject: [PATCH 08/84] adds a reflection api --- nova/api/easy.py | 52 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/nova/api/easy.py b/nova/api/easy.py index 1be52069fe2b..0e4f8a8923ef 100644 --- a/nova/api/easy.py +++ b/nova/api/easy.py @@ -31,6 +31,7 @@ The general flow of a request is: """ +import inspect import urllib import routes @@ -102,6 +103,52 @@ class SundayMorning(wsgi.Router): mapper.connect('/%s/{action}' % route, controller=ServiceWrapper(EASY_ROUTES[route])) + +class Reflection(object): + def __init__(self): + self._methods = {} + + def _gather_methods(self): + methods = {} + for route, handler in EASY_ROUTES.iteritems(): + for k in dir(handler): + if k.startswith('_'): + continue + f = getattr(handler, k) + if not callable(f): + continue + + # bunch of ugly formatting stuff + argspec = inspect.getargspec(f) + args = [x for x in argspec[0] if x != 'self' and x != 'context'] + defaults = argspec[3] and argspec[3] or [] + args_r = list(reversed(args)) + defaults_r = list(reversed(defaults)) + args_out = [] + while args_r: + if defaults_r: + args_out.append((args_r.pop(0), defaults_r.pop(0))) + else: + args_out.append(str(args_r.pop(0))) + + methods['/%s/%s' % (route, k)] = { + 'name': k, + 'args': list(reversed(args_out))} + return methods + + def get_methods(self, context): + if not self._methods: + self._methods = self._gather_methods() + + method_list = self._methods.keys() + method_list.sort() + return {'methods': method_list} + + def get_method_info(self, context, method): + if not self._methods: + self._methods = self._gather_methods() + return self._methods[method] + class ServiceWrapper(wsgi.Controller): def __init__(self, service_handle): @@ -148,7 +195,7 @@ class Proxy(object): def __getattr__(self, key): if self.prefix is None: - return self.__class__(self.app, key) + return self.__class__(self.app, prefix=key) def _wrapper(context, **kwargs): return self.__do_request('/%s/%s' % (self.prefix, key), @@ -156,6 +203,3 @@ class Proxy(object): **kwargs) _wrapper.func_name = key return _wrapper - - - From ef8e4495f5ed195a08be6c02b3eb3326f6403bb6 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 23 Dec 2010 16:56:21 -0600 Subject: [PATCH 09/84] updated the xenstore methods to reflect that they write to the param record of xenstore, not the actual xenstore itself. --- nova/virt/xenapi/vmops.py | 142 ++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 60 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 7bff47507c40..0e22ce306ea4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -18,7 +18,10 @@ Management class for VM-related functions (spawn, reboot, etc). """ +import json import logging +import random +import uuid from nova import db from nova import context @@ -53,13 +56,12 @@ class VMOps(object): """Create VM instance""" vm = VMHelper.lookup(self._session, instance.name) if vm is not None: - raise exception.Duplicate(_('Attempted to create' - ' non-unique name %s') % instance.name) + raise exception.Duplicate(_('Attempted to create non-unique name %s') + % instance.name) bridge = db.network_get_by_instance(context.get_admin_context(), instance['id'])['bridge'] - network_ref = \ - NetworkHelper.find_network_with_bridge(self._session, bridge) + network_ref = NetworkHelper.find_network_with_bridge(self._session, bridge) user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) @@ -104,16 +106,6 @@ class VMOps(object): timer.f = _wait_for_boot return timer.start(interval=0.5, now=True) - def reboot(self, instance): - """Reboot VM instance""" - instance_name = instance.name - vm = VMHelper.lookup(self._session, instance_name) - if vm is None: - raise exception.NotFound(_('instance not' - ' found %s') % instance_name) - task = self._session.call_xenapi('Async.VM.clean_reboot', vm) - self._session.wait_for_task(instance.id, task) - def _get_vm_opaque_ref(self, instance_or_vm): try: instance_name = instance_or_vm.name @@ -122,15 +114,21 @@ class VMOps(object): # A vm opaque ref was passed vm = instance_or_vm if vm is None: - raise Exception('instance not present %s' % instance_name) + raise Exception(_('Instance not present %s') % instance_name) return vm + def reboot(self, instance): + """Reboot VM instance""" + vm = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.clean_reboot', vm) + self._session.wait_for_task(instance.id, task) + def reset_root_password(self, instance): """Reset the root/admin password on the VM instance""" - self.add_to_xenstore(instance, {"reset_root_password": "requested"}) - self.add_to_xenstore(instance, {"TEST": "OMG!"}) + self.add_to_param_xenstore(instance, "reset_root_password", "requested") + self.add_to_param_xenstore(instance, "TEST", "OMG!") import time - self.add_to_xenstore(instance, {"timestamp": time.ctime()}) + self.add_to_param_xenstore(instance, "timestamp", time.ctime()) def destroy(self, instance): @@ -173,38 +171,27 @@ class VMOps(object): def pause(self, instance, callback): """Pause VM instance""" - instance_name = instance.name - vm = VMHelper.lookup(self._session, instance_name) - if vm is None: - raise exception.NotFound(_('Instance not' - ' found %s') % instance_name) + vm = self._get_vm_opaque_ref(instance) task = self._session.call_xenapi('Async.VM.pause', vm) self._wait_with_callback(instance.id, task, callback) def unpause(self, instance, callback): """Unpause VM instance""" - instance_name = instance.name - vm = VMHelper.lookup(self._session, instance_name) - if vm is None: - raise exception.NotFound(_('Instance not' - ' found %s') % instance_name) + vm = self._get_vm_opaque_ref(instance) task = self._session.call_xenapi('Async.VM.unpause', vm) self._wait_with_callback(instance.id, task, callback) def get_info(self, instance_id): """Return data about VM instance""" - vm = VMHelper.lookup(self._session, instance_id) + vm = VMHelper.lookup_blocking(self._session, instance_id) if vm is None: - raise exception.NotFound(_('Instance not' - ' found %s') % instance_id) + raise Exception(_('Instance not present %s') % instance_id) rec = self._session.get_xenapi().VM.get_record(vm) return VMHelper.compile_info(rec) def get_diagnostics(self, instance_id): """Return data about VM diagnostics""" - vm = VMHelper.lookup(self._session, instance_id) - if vm is None: - raise exception.NotFound(_("Instance not found %s") % instance_id) + vm = self._get_vm_opaque_ref(instance) rec = self._session.get_xenapi().VM.get_record(vm) return VMHelper.compile_diagnostics(self._session, rec) @@ -213,40 +200,75 @@ class VMOps(object): # TODO: implement this to fix pylint! return 'FAKE CONSOLE OUTPUT of instance' - def read_from_xenstore(self, instance_or_vm, keys=None): + def dh_keyinit(self, instance): + """Initiates a Diffie-Hellman (or, more precisely, a + Diffie-Hellman-Merkle) key exchange with the agent. It will + compute one side of the exchange and write it to xenstore. + When a response is received, it will then compute the shared + secret key, which is returned. + NOTE: the base and prime are pre-set; this may change in + the future. + """ + base = 5 + prime = 162259276829213363391578010288127 + secret_int = random.randint(100,1000) + val = (base ** secret_int) % prime + msgname = str(uuid.uuid4()) + key = "/data/host/%s" % msgname + self.add_to_param_xenstore(instance, key=key, + value={"name": "keyinit", "value": val}) + + def read_partial_from_param_xenstore(self, instance_or_vm, key_prefix): + """Returns a dict of all the keys in the xenstore for the given instance + that begin with the key_prefix. + """ + data = self.read_from_param_xenstore(instance_or_vm) + badkeys = [k for k in data.keys() + if not k.startswith(key_prefix)] + for badkey in badkeys: + del data[badkey] + return data + + def read_from_param_xenstore(self, instance_or_vm, keys=None): """Returns the xenstore data for the specified VM instance as - a dict. Accepts an optional list of keys; if the list of keys is - passed, the returned dict is filtered to only return the values + a dict. Accepts an optional key or list of keys; if a value for 'keys' + is passed, the returned dict is filtered to only return the values for those keys. """ vm = self._get_vm_opaque_ref(instance_or_vm) - ret = self._session.call_xenapi_request('VM.get_xenstore_data', (vm, )) - if keys: - allkeys = set(ret.keys()) - badkeys = allkeys.difference(keys) - for k in badkeys: - del ret[k] + data = self._session.call_xenapi_request('VM.get_xenstore_data', (vm, )) + ret = {} + if keys is None: + keys = data.keys() + elif isinstance(keys, basestring): + keys = [keys] + for key in keys: + raw = data.get(key) + if raw: + ret[key] = json.loads(raw) + else: + ret[key] = raw return ret - def add_to_xenstore(self, instance_or_vm, mapping): - """Takes a dict and adds it to the xenstore record for - the given vm instance. Existing data is preserved, but any - existing values for the mapping's keys are overwritten. - """ + def add_to_param_xenstore(self, instance_or_vm, key, val): + """Takes a key/value pair and adds it to the xenstore record + for the given vm instance. If the key exists in xenstore, it is + overwritten""" vm = self._get_vm_opaque_ref(instance_or_vm) - current_data = self.read_from_xenstore(vm) - current_data.update(mapping) - self.write_to_xenstore(vm, current_data) + self.remove_from_param_xenstore(instance_or_vm, key) + jsonval = json.dumps(val) + self._session.call_xenapi_request('VM.add_to_xenstore_data', + (vm, key, jsonval)) - def write_to_xenstore(self, instance_or_vm, mapping): - """Takes a dict and writes it to the xenstore record for - the given vm instance. Any existing data is overwritten. + def write_to_param_xenstore(self, instance_or_vm, mapping): + """Takes a dict and writes each key/value pair to the xenstore + record for the given vm instance. Any existing data for those + keys is overwritten. """ - vm = self._get_vm_opaque_ref(instance_or_vm) - self._session.call_xenapi_request('VM.set_xenstore_data', - (vm, mapping)) + for k, v in mapping.iteritems(): + self.add_to_param_xenstore(instance_or_vm, k, v) - def remove_from_xenstore(self, instance_or_vm, key_or_keys): + def remove_from_param_xenstore(self, instance_or_vm, key_or_keys): """Takes either a single key or a list of keys and removes them from the xenstore data for the given VM. If the key doesn't exist, the request is ignored. @@ -270,6 +292,6 @@ class VMOps(object): for key in keys: self._session.call_xenapi_request('VM.remove_from_xenstore_data', (vm, key)) - def clear_xenstore(self, instance_or_vm): + def clear_param_xenstore(self, instance_or_vm): """Removes all data from the xenstore record for this VM.""" - self.write_to_xenstore(instance_or_vm, {}) + self.write_to_param_xenstore(instance_or_vm, {}) From c5c58cb20def79401a374f863983a343139b53f3 Mon Sep 17 00:00:00 2001 From: "NTT PF Lab." Date: Fri, 24 Dec 2010 20:38:49 +0900 Subject: [PATCH 10/84] Support IPv6 --- bin/nova-manage | 12 +- contrib/boto_v6/__init__.py | 37 +++++ contrib/boto_v6/ec2/__init__.py | 0 contrib/boto_v6/ec2/connection.py | 41 ++++++ contrib/boto_v6/ec2/instance.py | 33 +++++ contrib/nova.sh | 6 + nova/api/ec2/cloud.py | 9 +- nova/db/api.py | 9 ++ nova/db/sqlalchemy/api.py | 23 +++ nova/db/sqlalchemy/models.py | 4 + nova/network/linux_net.py | 88 ++++++++++++ nova/network/manager.py | 30 +++- nova/test.py | 3 +- nova/tests/api_unittest.py | 67 +++++++++ nova/tests/network_unittest.py | 21 +++ nova/utils.py | 39 +++++ nova/virt/libvirt.qemu.xml.template | 1 + nova/virt/libvirt.uml.xml.template | 1 + nova/virt/libvirt_conn.py | 68 +++++++-- smoketests/admin_smoketests.py | 3 +- smoketests/base.py | 17 ++- smoketests/flags.py | 3 +- smoketests/public_network_smoketests.py | 180 ++++++++++++++++++++++++ smoketests/user_smoketests.py | 39 +++-- 24 files changed, 702 insertions(+), 32 deletions(-) create mode 100644 contrib/boto_v6/__init__.py create mode 100644 contrib/boto_v6/ec2/__init__.py create mode 100644 contrib/boto_v6/ec2/connection.py create mode 100644 contrib/boto_v6/ec2/instance.py create mode 100644 smoketests/public_network_smoketests.py diff --git a/bin/nova-manage b/bin/nova-manage index 0c1b621ed72f..e0f6f9323cbc 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -87,7 +87,7 @@ flags.DECLARE('num_networks', 'nova.network.manager') flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') - +flags.DECLARE('fixed_range_v6', 'nova.network.manager') class VpnCommands(object): """Class for managing VPNs.""" @@ -411,11 +411,11 @@ class NetworkCommands(object): """Class for managing networks.""" def create(self, fixed_range=None, num_networks=None, - network_size=None, vlan_start=None, vpn_start=None): + network_size=None, vlan_start=None, vpn_start=None,fixed_range_v6=None): """Creates fixed ips for host by range arguments: [fixed_range=FLAG], [num_networks=FLAG], [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG]""" + [vpn_start=FLAG],[fixed_range_v6=FLAG]""" if not fixed_range: fixed_range = FLAGS.fixed_range if not num_networks: @@ -426,11 +426,15 @@ class NetworkCommands(object): vlan_start = FLAGS.vlan_start if not vpn_start: vpn_start = FLAGS.vpn_start + if not fixed_range_v6: + fixed_range_v6 = FLAGS.fixed_range_v6 net_manager = utils.import_object(FLAGS.network_manager) net_manager.create_networks(context.get_admin_context(), fixed_range, int(num_networks), int(network_size), int(vlan_start), - int(vpn_start)) + int(vpn_start),fixed_range_v6) + + CATEGORIES = [ ('user', UserCommands), diff --git a/contrib/boto_v6/__init__.py b/contrib/boto_v6/__init__.py new file mode 100644 index 000000000000..9fec157f178d --- /dev/null +++ b/contrib/boto_v6/__init__.py @@ -0,0 +1,37 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +def connect_ec2(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.connection.EC2Connection` + :return: A connection to Amazon's EC2 + """ + from boto_v6.ec2.connection import EC2ConnectionV6 + return EC2ConnectionV6(aws_access_key_id, aws_secret_access_key, **kwargs) diff --git a/contrib/boto_v6/ec2/__init__.py b/contrib/boto_v6/ec2/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/contrib/boto_v6/ec2/connection.py b/contrib/boto_v6/ec2/connection.py new file mode 100644 index 000000000000..151b76a556c8 --- /dev/null +++ b/contrib/boto_v6/ec2/connection.py @@ -0,0 +1,41 @@ +''' +Created on 2010/12/20 + +@author: Nachi Ueno +''' +import boto +import boto.ec2 +from boto_v6.ec2.instance import ReservationV6 + + +class EC2ConnectionV6(boto.ec2.EC2Connection): + ''' + EC2Connection for OpenStack IPV6 mode + ''' + def get_all_instances(self, instance_ids=None, filters=None): + """ + Retrieve all the instances associated with your account. + + :type instance_ids: list + :param instance_ids: A list of strings of instance IDs + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.instance.Reservation` + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeInstances', params, + [('item', ReservationV6)]) diff --git a/contrib/boto_v6/ec2/instance.py b/contrib/boto_v6/ec2/instance.py new file mode 100644 index 000000000000..255114935e42 --- /dev/null +++ b/contrib/boto_v6/ec2/instance.py @@ -0,0 +1,33 @@ +''' +Created on 2010/12/20 + +@author: Nachi Ueno +''' +import boto +from boto.resultset import ResultSet +from boto.ec2.instance import Reservation +from boto.ec2.instance import Group +from boto.ec2.instance import Instance + + +class ReservationV6(Reservation): + def startElement(self, name, attrs, connection): + if name == 'instancesSet': + self.instances = ResultSet([('item', InstanceV6)]) + return self.instances + elif name == 'groupSet': + self.groups = ResultSet([('item', Group)]) + return self.groups + else: + return None + + +class InstanceV6(Instance): + def __init__(self, connection=None): + Instance.__init__(self, connection) + self.public_dns_name_v6 = None + + def endElement(self, name, value, connection): + Instance.endElement(self, name, value, connection) + if name == 'dnsNameV6': + self.dns_name_v6 = value diff --git a/contrib/nova.sh b/contrib/nova.sh index 30df4edb654a..bc46740ada87 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -85,6 +85,10 @@ if [ "$CMD" == "install" ]; then sudo apt-get install -y python-twisted python-sqlalchemy python-mox python-greenlet python-carrot sudo apt-get install -y python-daemon python-eventlet python-gflags python-tornado python-ipy sudo apt-get install -y python-libvirt python-libxml2 python-routes +#For IPV6 + sudo apt-get install -y python-netaddr + sudo apt-get install -y radvd + if [ "$USE_MYSQL" == 1 ]; then cat < /proc/sys/net/ipv6/conf/all/forwarding"') + _execute('sudo bash -c ' + + '"echo 0 > /proc/sys/net/ipv6/conf/all/accept_ra"') def bind_floating_ip(floating_ip): @@ -158,6 +164,10 @@ def ensure_bridge(bridge, interface, net_attrs=None): net_attrs['gateway'], net_attrs['broadcast'], net_attrs['netmask'])) + if(FLAGS.use_ipv6): + _execute("sudo ifconfig %s add %s up" % \ + (bridge, + net_attrs['cidr_v6'])) else: _execute("sudo ifconfig %s up" % bridge) _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) @@ -213,6 +223,50 @@ def update_dhcp(context, network_id): _execute(command, addl_env=env) +def update_ra(context, network_id): + network_ref = db.network_get(context, network_id) + + conffile = _ra_file(network_ref['bridge'], 'conf') + with open(conffile, 'w') as f: + conf_str = """ +interface %s +{ + AdvSendAdvert on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 10; + prefix %s + { + AdvOnLink on; + AdvAutonomous on; + }; +}; +""" % (network_ref['bridge'], network_ref['cidr_v6']) + f.write(conf_str) + + # Make sure dnsmasq can actually read it (it setuid()s to "nobody") + os.chmod(conffile, 0644) + + pid = _ra_pid_for(network_ref['bridge']) + + # if dnsmasq is already running, then tell it to reload + if pid: + out, _err = _execute('cat /proc/%d/cmdline' + % pid, check_exit_code=False) + if conffile in out: + try: + _execute('sudo kill -HUP %d' % pid) + return + except Exception as exc: # pylint: disable-msg=W0703 + logging.debug("Hupping radvd threw %s", exc) + else: + logging.debug("Pid %d is stale, relaunching radvd", pid) + command = _ra_cmd(network_ref) + _execute(command) + db.network_update(context, network_id, + {"ra_server": + utils.get_my_linklocal(network_ref['bridge'])}) + + def _host_dhcp(fixed_ip_ref): """Return a host string for an address""" instance_ref = fixed_ip_ref['instance'] @@ -268,6 +322,15 @@ def _dnsmasq_cmd(net): return ''.join(cmd) +def _ra_cmd(net): + """Builds dnsmasq command""" + cmd = ['sudo -E radvd', +# ' -u nobody', + ' -C %s' % _ra_file(net['bridge'], 'conf'), + ' -p %s' % _ra_file(net['bridge'], 'pid')] + return ''.join(cmd) + + def _stop_dnsmasq(network): """Stops the dnsmasq instance for a given network""" pid = _dnsmasq_pid_for(network) @@ -289,6 +352,16 @@ def _dhcp_file(bridge, kind): kind)) +def _ra_file(bridge, kind): + """Return path to a pid, leases or conf file for a bridge""" + + if not os.path.exists(FLAGS.networks_path): + os.makedirs(FLAGS.networks_path) + return os.path.abspath("%s/nova-ra-%s.%s" % (FLAGS.networks_path, + bridge, + kind)) + + def _dnsmasq_pid_for(bridge): """Returns the pid for prior dnsmasq instance for a bridge @@ -302,3 +375,18 @@ def _dnsmasq_pid_for(bridge): if os.path.exists(pid_file): with open(pid_file, 'r') as f: return int(f.read()) + + +def _ra_pid_for(bridge): + """Returns the pid for prior dnsmasq instance for a bridge + + Returns None if no pid file exists + + If machine has rebooted pid might be incorrect (caller should check) + """ + + pid_file = _ra_file(bridge, 'pid') + + if os.path.exists(pid_file): + with open(pid_file, 'r') as f: + return int(f.read()) diff --git a/nova/network/manager.py b/nova/network/manager.py index a7298b47f0c2..ceea6966fe5d 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -80,6 +80,7 @@ flags.DEFINE_integer('network_size', 256, flags.DEFINE_string('floating_range', '4.4.4.0/24', 'Floating IP address block') flags.DEFINE_string('fixed_range', '10.0.0.0/8', 'Fixed IP address block') +flags.DEFINE_string('fixed_range_v6', 'fd00::/48', 'Fixed IPv6 address block') flags.DEFINE_integer('cnt_vpn_clients', 5, 'Number of addresses reserved for vpn clients') flags.DEFINE_string('network_driver', 'nova.network.linux_net', @@ -88,6 +89,8 @@ flags.DEFINE_bool('update_dhcp_on_disassociate', False, 'Whether to update dhcp when fixed_ip is disassociated') flags.DEFINE_integer('fixed_ip_disassociate_timeout', 600, 'Seconds after which a deallocated ip is disassociated') +flags.DEFINE_bool('use_ipv6', True, + 'use the ipv6') class AddressAlreadyAllocated(exception.Error): @@ -217,7 +220,7 @@ class NetworkManager(manager.Manager): """Get the network for the current context.""" raise NotImplementedError() - def create_networks(self, context, num_networks, network_size, + def create_networks(self, context, num_networks, network_size, cidr_v6, *args, **kwargs): """Create networks based on parameters.""" raise NotImplementedError() @@ -307,9 +310,11 @@ class FlatManager(NetworkManager): pass def create_networks(self, context, cidr, num_networks, network_size, - *args, **kwargs): + cidr_v6, *args, **kwargs): """Create networks based on parameters.""" fixed_net = IPy.IP(cidr) + fixed_net_v6 = IPy.IP(cidr_v6) + significant_bits_v6 = 64 for index in range(num_networks): start = index * network_size significant_bits = 32 - int(math.log(network_size, 2)) @@ -322,7 +327,13 @@ class FlatManager(NetworkManager): net['gateway'] = str(project_net[1]) net['broadcast'] = str(project_net.broadcast()) net['dhcp_start'] = str(project_net[2]) + + if(FLAGS.use_ipv6): + cidr_v6 = "%s/%s" % (fixed_net_v6[0], significant_bits_v6) + net['cidr_v6'] = cidr_v6 + network_ref = self.db.network_create_safe(context, net) + if network_ref: self._create_fixed_ips(context, network_ref['id']) @@ -466,12 +477,16 @@ class VlanManager(NetworkManager): pass def create_networks(self, context, cidr, num_networks, network_size, - vlan_start, vpn_start): + vlan_start, vpn_start, cidr_v6): """Create networks based on parameters.""" fixed_net = IPy.IP(cidr) + fixed_net_v6 = IPy.IP(cidr_v6) + network_size_v6 = 1 << 64 + significant_bits_v6 = 64 for index in range(num_networks): vlan = vlan_start + index start = index * network_size + start_v6 = index * network_size_v6 significant_bits = 32 - int(math.log(network_size, 2)) cidr = "%s/%s" % (fixed_net[start], significant_bits) project_net = IPy.IP(cidr) @@ -484,6 +499,13 @@ class VlanManager(NetworkManager): net['dhcp_start'] = str(project_net[3]) net['vlan'] = vlan net['bridge'] = 'br%s' % vlan + if(FLAGS.use_ipv6): + cidr_v6 = "%s/%s" % ( + fixed_net_v6[start_v6], + significant_bits_v6 + ) + net['cidr_v6'] = cidr_v6 + # NOTE(vish): This makes ports unique accross the cloud, a more # robust solution would be to make them unique per ip net['vpn_public_port'] = vpn_start + index @@ -506,6 +528,8 @@ class VlanManager(NetworkManager): network_ref['bridge'], network_ref) self.driver.update_dhcp(context, network_id) + if(FLAGS.use_ipv6): + self.driver.update_ra(context, network_id) @property def _bottom_reserved_ips(self): diff --git a/nova/test.py b/nova/test.py index 5c2a72819683..ee2fc2720a45 100644 --- a/nova/test.py +++ b/nova/test.py @@ -70,7 +70,8 @@ class TrialTestCase(unittest.TestCase): FLAGS.fixed_range, 5, 16, FLAGS.vlan_start, - FLAGS.vpn_start) + FLAGS.vpn_start, + FLAGS.fixed_range_v6) # emulate some of the mox stuff, we can't use the metaclass # because it screws with our generators diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 33d4cb294e13..a508235c43bc 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -24,6 +24,7 @@ import httplib import random import StringIO import webob +import logging from nova import context from nova import flags @@ -265,6 +266,72 @@ class ApiEc2TestCase(test.TrialTestCase): return + def test_authorize_revoke_security_group_cidr_v6(self): + """ + Test that we can add and remove CIDR based rules + to a security group for IPv6 + """ + self.expect_http() + self.mox.ReplayAll() + user = self.manager.create_user('fake', 'fake', 'fake') + project = self.manager.create_project('fake', 'fake', 'fake') + + # At the moment, you need both of these to actually be netadmin + self.manager.add_role('fake', 'netadmin') + project.add_role('fake', 'netadmin') + + security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") + for x in range(random.randint(4, 8))) + + group = self.ec2.create_security_group(security_group_name, + 'test group') + + self.expect_http() + self.mox.ReplayAll() + group.connection = self.ec2 + + group.authorize('tcp', 80, 81, '::/0') + + self.expect_http() + self.mox.ReplayAll() + + rv = self.ec2.get_all_security_groups() + # I don't bother checkng that we actually find it here, + # because the create/delete unit test further up should + # be good enough for that. + for group in rv: + if group.name == security_group_name: + self.assertEquals(len(group.rules), 1) + self.assertEquals(int(group.rules[0].from_port), 80) + self.assertEquals(int(group.rules[0].to_port), 81) + self.assertEquals(len(group.rules[0].grants), 1) + self.assertEquals(str(group.rules[0].grants[0]), '::/0') + + self.expect_http() + self.mox.ReplayAll() + group.connection = self.ec2 + + group.revoke('tcp', 80, 81, '::/0') + + self.expect_http() + self.mox.ReplayAll() + + self.ec2.delete_security_group(security_group_name) + + self.expect_http() + self.mox.ReplayAll() + group.connection = self.ec2 + + rv = self.ec2.get_all_security_groups() + + self.assertEqual(len(rv), 1) + self.assertEqual(rv[0].name, 'default') + + self.manager.delete_project(project) + self.manager.delete_user(user) + + return + def test_authorize_revoke_security_group_foreign_group(self): """ Test that we can grant and revoke another security group access diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 6f4705719c9b..0a4b50e96ec1 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -97,6 +97,27 @@ class NetworkTestCase(test.TrialTestCase): self.context.project_id = self.projects[project_num].id self.network.deallocate_fixed_ip(self.context, address) + def test_private_ipv6(self): + """Make sure ipv6 is OK""" + if FLAGS.use_ipv6: + instance_ref = self._create_instance(1) + network_ref = db.project_get_network( + self.context, + self.context.project_id) + address_v6 = db.instance_get_fixed_address_v6( + self.context, + instance_ref['id']) + self.assertEqual(instance_ref['mac_address'], + utils.to_mac(address_v6)) + instance_ref2 = db.fixed_ip_get_instance_v6( + self.context, + address_v6) + self.assertEqual(instance_ref['id'], instance_ref2['id']) + self.assertEqual(address_v6, + utils.to_global_ipv6( + network_ref['cidr_v6'], + instance_ref['mac_address'])) + def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" # TODO(vish): better way of adding floating ips diff --git a/nova/utils.py b/nova/utils.py index 142584df8b26..211a2cb75f8c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -30,6 +30,8 @@ import subprocess import socket import sys from xml.sax import saxutils +import re +import netaddr from twisted.internet.threads import deferToThread @@ -45,10 +47,14 @@ TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" def import_class(import_str): """Returns a class from a string including module and class""" mod_str, _sep, class_str = import_str.rpartition('.') + logging.debug(import_str) try: __import__(mod_str) return getattr(sys.modules[mod_str], class_str) except (ImportError, ValueError, AttributeError): + logging.debug(ImportError) + logging.debug(ValueError) + logging.debug(AttributeError) raise exception.NotFound('Class %s cannot be found' % class_str) @@ -165,6 +171,39 @@ def get_my_ip(): return "127.0.0.1" +def get_my_linklocal(interface): + if getattr(FLAGS, 'fake_tests', None): + return 'fe00::' + try: + if_str = execute("ifconfig %s" % interface) + condition = "\s+inet6\s+addr:\s+([0-9a-f:]+/\d+)\s+Scope:Link" + links = [re.search(condition, x) for x in if_str[0].split('\n')] + address = [w.group(1) for w in links if w is not None] + if address[0] is not None: + return address[0] + else: + return None + except RuntimeError as ex: + logging.warn("Couldn't get Link Local IP of %s :%s", interface, ex) + return None + + +def to_global_ipv6(prefix, mac): + mac64 = netaddr.EUI(mac).eui64().words + int_addr = int(''.join(['%02x' % i for i in mac64]), 16) + mac64_addr = netaddr.IPAddress(int_addr) + maskIP = netaddr.IPNetwork(prefix).ip + return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).format() + + +def to_mac(ipv6_address): + address = netaddr.IPAddress(ipv6_address) + mask1 = netaddr.IPAddress("::ffff:ffff:ffff:ffff") + mask2 = netaddr.IPAddress("::0200:0:0:0") + mac64 = netaddr.EUI(int(address & mask1 ^ mask2)).words + return ":".join(["%02x" % i for i in mac64[0:3] + mac64[5:8]]) + + def isotime(at=None): if not at: at = datetime.datetime.utcnow() diff --git a/nova/virt/libvirt.qemu.xml.template b/nova/virt/libvirt.qemu.xml.template index 2538b1adefa9..0ffe17e8dda5 100644 --- a/nova/virt/libvirt.qemu.xml.template +++ b/nova/virt/libvirt.qemu.xml.template @@ -23,6 +23,7 @@ + diff --git a/nova/virt/libvirt.uml.xml.template b/nova/virt/libvirt.uml.xml.template index bb8b479113de..0d355b81cf9b 100644 --- a/nova/virt/libvirt.uml.xml.template +++ b/nova/virt/libvirt.uml.xml.template @@ -17,6 +17,7 @@ + diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 18085089f3b5..71d9f781dd15 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -514,6 +514,8 @@ class LibvirtConnection(object): instance['id']) # Assume that the gateway also acts as the dhcp server. dhcp_server = network['gateway'] + #TODO ipv6 + ra_server = network['ra_server'] xml_info = {'type': FLAGS.libvirt_type, 'name': instance['name'], 'basepath': os.path.join(FLAGS.instances_path, @@ -523,11 +525,13 @@ class LibvirtConnection(object): 'bridge_name': network['bridge'], 'mac_address': instance['mac_address'], 'ip_address': ip_address, - 'dhcp_server': dhcp_server} + 'dhcp_server': dhcp_server, + 'ra_server': ra_server} if rescue: libvirt_xml = self.rescue_xml % xml_info else: libvirt_xml = self.libvirt_xml % xml_info + logging.debug('instance %s: finished toXML method', instance['name']) return libvirt_xml @@ -701,6 +705,7 @@ class NWFilterFirewall(object): + ''' @@ -722,6 +727,14 @@ class NWFilterFirewall(object): ''' + nova_ra_filter = ''' + d707fa71-4fb5-4b27-9ab7-ba5ca19c8804 + + + + ''' + def nova_base_ipv4_filter(self): retval = "" for protocol in ['tcp', 'udp', 'icmp']: @@ -736,13 +749,13 @@ class NWFilterFirewall(object): def nova_base_ipv6_filter(self): retval = "" - for protocol in ['tcp', 'udp', 'icmp']: + for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: for direction, action, priority in [('out', 'accept', 399), ('inout', 'drop', 400)]: retval += """ - <%s-ipv6 /> + <%s /> """ % (action, direction, - priority, protocol) + priority, protocol) retval += '' return retval @@ -755,6 +768,15 @@ class NWFilterFirewall(object): retval += '' return retval + def nova_project_filter_v6(self, project, net, mask): + retval = "" % project + for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: + retval += """ + <%s srcipaddr='%s' srcipmask='%s' /> + """ % (protocol, net, mask) + retval += '' + return retval + def _define_filter(self, xml): if callable(xml): xml = xml() @@ -766,6 +788,11 @@ class NWFilterFirewall(object): net = IPy.IP(cidr) return str(net.net()), str(net.netmask()) + @staticmethod + def _get_ip_version(cidr): + net = IPy.IP(cidr) + return int(net.version()) + @defer.inlineCallbacks def setup_nwfilters_for_instance(self, instance): """ @@ -777,6 +804,7 @@ class NWFilterFirewall(object): yield self._define_filter(self.nova_base_ipv4_filter) yield self._define_filter(self.nova_base_ipv6_filter) yield self._define_filter(self.nova_dhcp_filter) + yield self._define_filter(self.nova_ra_filter) yield self._define_filter(self.nova_base_filter) nwfilter_xml = "\n" \ @@ -787,12 +815,22 @@ class NWFilterFirewall(object): network_ref = db.project_get_network(context.get_admin_context(), instance['project_id']) net, mask = self._get_net_and_mask(network_ref['cidr']) + if(FLAGS.use_ipv6): + net_v6, mask_v6 = self._get_net_and_mask( + network_ref['cidr_v6']) project_filter = self.nova_project_filter(instance['project_id'], net, mask) yield self._define_filter(project_filter) - nwfilter_xml += " \n" % \ instance['project_id'] + if(FLAGS.use_ipv6): + project_filter_v6 = self.nova_project_filter_v6( + instance['project_id'], + net_v6, mask_v6) + yield self._define_filter(project_filter_v6) + nwfilter_xml += \ + " \n" % \ + instance['project_id'] for security_group in instance.security_groups: yield self.ensure_security_group_filter(security_group['id']) @@ -812,12 +850,21 @@ class NWFilterFirewall(object): security_group = db.security_group_get(context.get_admin_context(), security_group_id) rule_xml = "" + version = 4 + v6protocol = {'tcp':'tcp-ipv6', 'udp':'udp-ipv6', 'icmp':'icmpv6'} for rule in security_group.rules: rule_xml += "" if rule.cidr: + version = self._get_ip_version(rule.cidr) net, mask = self._get_net_and_mask(rule.cidr) - rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ - (rule.protocol, net, mask) + if(FLAGS.use_ipv6 and version == 6): + rule_xml += "<%s " % v6protocol[rule.protocol] + rule_xml += "srcipaddr='%s' " % net + rule_xml += "srcipmask='%s' " % mask + else: + rule_xml += "<%s " % rule.protocol + rule_xml += "srcipaddr='%s' " % net + rule_xml += "srcipmask='%s' " % mask if rule.protocol in ['tcp', 'udp']: rule_xml += "dstportstart='%s' dstportend='%s' " % \ (rule.from_port, rule.to_port) @@ -832,6 +879,9 @@ class NWFilterFirewall(object): rule_xml += '/>\n' rule_xml += "\n" - xml = "%s" % \ - (security_group_id, rule_xml,) + xml = " \n %s' % (cmd, output) @@ -130,6 +143,7 @@ class SmokeTestCase(unittest.TestCase): raise Exception(output) return True + def run_tests(suites): argv = FLAGS(sys.argv) @@ -151,4 +165,3 @@ def run_tests(suites): else: for suite in suites.itervalues(): unittest.TextTestRunner(verbosity=2).run(suite) - diff --git a/smoketests/flags.py b/smoketests/flags.py index ae4d0950850d..35f432a77800 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -33,6 +33,7 @@ DEFINE_bool = DEFINE_bool # __GLOBAL FLAGS ONLY__ # Define any app-specific flags in their own files, docs at: # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 + DEFINE_string('region', 'nova', 'Region to use') DEFINE_string('test_image', 'ami-tiny', 'Image to use for launch tests') - +DEFINE_string('use_ipv6', True, 'use the ipv6 or not') diff --git a/smoketests/public_network_smoketests.py b/smoketests/public_network_smoketests.py new file mode 100644 index 000000000000..bfc2b20ba103 --- /dev/null +++ b/smoketests/public_network_smoketests.py @@ -0,0 +1,180 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import socket +import sys +import time +import unittest + +from smoketests import flags +from smoketests import base +from smoketests import user_smoketests + +#Note that this test should run from +#public network (outside of private network segments) +#Please set EC2_URL correctly +#You should use admin account in this test + +FLAGS = flags.FLAGS + +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_KEY2 = '%s_key2' % TEST_PREFIX +TEST_DATA = {} + + +class InstanceTestsFromPublic(user_smoketests.UserSmokeTestCase): + def test_001_can_create_keypair(self): + key = self.create_key_pair(self.conn, TEST_KEY) + self.assertEqual(key.name, TEST_KEY) + + def test_002_security_group(self): + security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") + for x in range(random.randint(4, 8))) + group = self.conn.create_security_group(security_group_name, + 'test group') + group.connection = self.conn + group.authorize('tcp', 22, 22, '0.0.0.0/0') + if FLAGS.use_ipv6: + group.authorize('tcp', 22, 22, '::/0') + + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + security_groups=[security_group_name], + instance_type='m1.tiny') + self.data['security_group_name'] = security_group_name + self.data['group'] = group + self.data['instance_id'] = reservation.instances[0].id + + def test_003_instance_with_group_runs_within_60_seconds(self): + reservations = self.conn.get_all_instances([self.data['instance_id']]) + instance = reservations[0].instances[0] + # allow 60 seconds to exit pending with IP + for x in xrange(60): + instance.update() + if instance.state == u'running': + break + time.sleep(1) + else: + self.fail('instance failed to start') + ip = reservations[0].instances[0].private_dns_name + self.failIf(ip == '0.0.0.0') + self.data['private_ip'] = ip + if FLAGS.use_ipv6: + ipv6 = reservations[0].instances[0].dns_name_v6 + self.failIf(ipv6 is None) + self.data['ip_v6'] = ipv6 + + def test_004_can_ssh_to_ipv6(self): + if FLAGS.use_ipv6: + for x in xrange(20): + try: + conn = self.connect_ssh( + self.data['ip_v6'], TEST_KEY) + conn.close() + except Exception as ex: + print ex + time.sleep(1) + else: + break + else: + self.fail('could not ssh to instance') + + def test_012_can_create_instance_with_keypair(self): + if 'instance_id' in self.data: + self.conn.terminate_instances([self.data['instance_id']]) + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + instance_type='m1.tiny') + self.assertEqual(len(reservation.instances), 1) + self.data['instance_id'] = reservation.instances[0].id + + def test_013_instance_runs_within_60_seconds(self): + reservations = self.conn.get_all_instances([self.data['instance_id']]) + instance = reservations[0].instances[0] + # allow 60 seconds to exit pending with IP + for x in xrange(60): + instance.update() + if instance.state == u'running': + break + time.sleep(1) + else: + self.fail('instance failed to start') + ip = reservations[0].instances[0].private_dns_name + self.failIf(ip == '0.0.0.0') + self.data['private_ip'] = ip + if FLAGS.use_ipv6: + ipv6 = reservations[0].instances[0].dns_name_v6 + self.failIf(ipv6 is None) + self.data['ip_v6'] = ipv6 + + def test_014_can_not_ping_private_ip(self): + for x in xrange(4): + # ping waits for 1 second + status, output = commands.getstatusoutput( + 'ping -c1 %s' % self.data['private_ip']) + if status == 0: + self.fail('can ping private ip from public network') + if FLAGS.use_ipv6: + status, output = commands.getstatusoutput( + 'ping6 -c1 %s' % self.data['ip_v6']) + if status == 0: + self.fail('can ping ipv6 from public network') + else: + pass + + def test_015_can_not_ssh_to_private_ip(self): + for x in xrange(1): + try: + conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) + conn.close() + except Exception: + time.sleep(1) + else: + self.fail('can ssh for ipv4 address from public network') + + if FLAGS.use_ipv6: + for x in xrange(1): + try: + conn = self.connect_ssh( + self.data['ip_v6'], TEST_KEY) + conn.close() + except Exception: + time.sleep(1) + else: + self.fail('can ssh for ipv6 address from public network') + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + security_group_name = self.data['security_group_name'] + group = self.data['group'] + if group: + group.revoke('tcp', 22, 22, '0.0.0.0/0') + if FLAGS.use_ipv6: + group.revoke('tcp', 22, 22, '::/0') + self.conn.delete_security_group(security_group_name) + if 'instance_id' in self.data: + self.conn.terminate_instances([self.data['instance_id']]) + +if __name__ == "__main__": + suites = {'instance': unittest.makeSuite(InstanceTestsFromPublic)} + sys.exit(base.run_tests(suites)) diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index d29e3aea32d7..8bb2ddebb2a8 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -37,7 +37,7 @@ flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', 'Local image file to use for bundling tests') -TEST_PREFIX = 'test%s' % int (random.random()*1000000) +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) TEST_BUCKET = '%s_bucket' % TEST_PREFIX TEST_KEY = '%s_key' % TEST_PREFIX TEST_DATA = {} @@ -71,7 +71,7 @@ class ImageTests(UserSmokeTestCase): def test_006_can_register_kernel(self): kernel_id = self.conn.register_image('%s/%s.manifest.xml' % - (TEST_BUCKET, FLAGS.bundle_kernel)) + (TEST_BUCKET, FLAGS.bundle_kernel)) self.assert_(kernel_id is not None) self.data['kernel_id'] = kernel_id @@ -83,7 +83,7 @@ class ImageTests(UserSmokeTestCase): time.sleep(1) else: print image.state - self.assert_(False) # wasn't available within 10 seconds + self.assert_(False) # wasn't available within 10 seconds self.assert_(image.type == 'machine') for i in xrange(10): @@ -92,7 +92,7 @@ class ImageTests(UserSmokeTestCase): break time.sleep(1) else: - self.assert_(False) # wasn't available within 10 seconds + self.assert_(False) # wasn't available within 10 seconds self.assert_(kernel.type == 'kernel') def test_008_can_describe_image_attribute(self): @@ -137,20 +137,23 @@ class InstanceTests(UserSmokeTestCase): self.data['instance_id'] = reservation.instances[0].id def test_003_instance_runs_within_60_seconds(self): - reservations = self.conn.get_all_instances([data['instance_id']]) + reservations = self.conn.get_all_instances([self.data['instance_id']]) instance = reservations[0].instances[0] # allow 60 seconds to exit pending with IP for x in xrange(60): instance.update() if instance.state == u'running': - break + break time.sleep(1) else: self.fail('instance failed to start') ip = reservations[0].instances[0].private_dns_name self.failIf(ip == '0.0.0.0') self.data['private_ip'] = ip - print self.data['private_ip'] + if FLAGS.use_ipv6: + ipv6 = reservations[0].instances[0].dns_name_v6 + self.failIf(ipv6 is None) + self.data['ip_v6'] = ipv6 def test_004_can_ping_private_ip(self): for x in xrange(120): @@ -159,6 +162,11 @@ class InstanceTests(UserSmokeTestCase): 'ping -c1 %s' % self.data['private_ip']) if status == 0: break + if FLAGS.use_ipv6: + status, output = commands.getstatusoutput( + 'ping6 -c1 %s' % self.data['ip_v6']) + if status == 0: + break else: self.fail('could not ping instance') @@ -174,6 +182,19 @@ class InstanceTests(UserSmokeTestCase): else: self.fail('could not ssh to instance') + if FLAGS.use_ipv6: + for x in xrange(30): + try: + conn = self.connect_ssh( + self.data['ip_v6'], TEST_KEY) + conn.close() + except Exception: + time.sleep(1) + else: + break + else: + self.fail('could not ssh to instance') + def test_006_can_allocate_elastic_ip(self): result = self.conn.allocate_address() self.assertTrue(hasattr(result, 'public_ip')) @@ -206,8 +227,8 @@ class InstanceTests(UserSmokeTestCase): def test_999_tearDown(self): self.delete_key_pair(self.conn, TEST_KEY) - if self.data.has_key('instance_id'): - self.conn.terminate_instances([data['instance_id']]) + if 'instance_id' in self.data: + self.conn.terminate_instances([self.data['instance_id']]) class VolumeTests(UserSmokeTestCase): From 8e1b74aa1c5a2f9113473eedc8e35b38b41445ea Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Mon, 27 Dec 2010 15:15:24 -0800 Subject: [PATCH 11/84] Added stack command-line tool --- bin/nova-easy-api | 61 +++++++++++++++ bin/stack | 145 ++++++++++++++++++++++++++++++++++++ nova/api/easy.py | 57 ++++++++++---- nova/compute/api.py | 6 ++ nova/tests/easy_unittest.py | 6 +- nova/utils.py | 2 +- nova/wsgi.py | 3 +- 7 files changed, 259 insertions(+), 21 deletions(-) create mode 100755 bin/nova-easy-api create mode 100755 bin/stack diff --git a/bin/nova-easy-api b/bin/nova-easy-api new file mode 100755 index 000000000000..e8e86b4fb41b --- /dev/null +++ b/bin/nova-easy-api @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# pylint: disable-msg=C0103 +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starter script for Nova Easy API.""" + +import gettext +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +gettext.install('nova', unicode=1) + +from nova import flags +from nova import utils +from nova import wsgi +from nova.api import easy +from nova.compute import api as compute_api + + +FLAGS = flags.FLAGS +flags.DEFINE_integer('easy_port', 8001, 'Easy API port') +flags.DEFINE_string('easy_host', '0.0.0.0', 'Easy API host') + +if __name__ == '__main__': + utils.default_flagfile() + FLAGS(sys.argv) + + easy.register_service('compute', compute_api.ComputeAPI()) + easy.register_service('reflect', easy.Reflection()) + router = easy.SundayMorning() + with_json = easy.JsonParamsMiddleware(router) + with_req = easy.ReqParamsMiddleware(with_json) + with_auth = easy.DelegatedAuthMiddleware(with_req) + + server = wsgi.Server() + server.start(with_auth, FLAGS.easy_port, host=FLAGS.easy_host) + server.wait() diff --git a/bin/stack b/bin/stack new file mode 100755 index 000000000000..284dbf4fc6f3 --- /dev/null +++ b/bin/stack @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""CLI for the Easy API.""" + +import eventlet +eventlet.monkey_patch() + +import os +import pprint +import sys +import textwrap +import urllib +import urllib2 + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +import gflags +from nova import utils + + +FLAGS = gflags.FLAGS +gflags.DEFINE_string('host', '127.0.0.1', 'Easy API host') +gflags.DEFINE_integer('port', 8001, 'Easy API host') +gflags.DEFINE_string('user', 'user1', 'Easy API username') +gflags.DEFINE_string('project', 'proj1', 'Easy API project') + + +USAGE = """usage: stack [options] [arg1=value arg2=value] + + `stack help` should output the list of available controllers + `stack ` should output the available methods for that controller + `stack help ` should do the same + `stack help ` should output info for a method +""" + + +def format_help(d): + """Format help text, keys are labels and values are descriptions.""" + indent = max([len(k) for k in d]) + out = [] + for k, v in d.iteritems(): + t = textwrap.TextWrapper(initial_indent=' %s ' % k.ljust(indent), + subsequent_indent=' ' * (indent + 6)) + out.extend(t.wrap(v)) + return out + + +def help_all(): + rv = do_request('reflect', 'get_controllers') + out = format_help(rv) + return (USAGE + str(FLAGS.MainModuleHelp()) + + '\n\nAvailable controllers:\n' + + '\n'.join(out) + '\n') + + +def help_controller(controller): + rv = do_request('reflect', 'get_methods') + methods = dict([(k.split('/')[2], v) for k, v in rv.iteritems() + if k.startswith('/%s' % controller)]) + return ('Available methods for %s:\n' % controller + + '\n'.join(format_help(methods))) + + +def help_method(controller, method): + rv = do_request('reflect', + 'get_method_info', + {'method': '/%s/%s' % (controller, method)}) + + sig = '%s(%s):' % (method, ', '.join(['='.join(x) for x in rv['args']])) + out = textwrap.wrap(sig, subsequent_indent=' ' * len('%s(' % method)) + out.append('\n' + rv['doc']) + return '\n'.join(out) + + +def do_request(controller, method, params=None): + if params: + data = urllib.urlencode(params) + else: + data = None + + url = 'http://%s:%s/%s/%s' % (FLAGS.host, FLAGS.port, controller, method) + headers = {'X-OpenStack-User': FLAGS.user, + 'X-OpenStack-Project': FLAGS.project} + + req = urllib2.Request(url, data, headers) + resp = urllib2.urlopen(req) + return utils.loads(resp.read()) + + +if __name__ == '__main__': + args = FLAGS(sys.argv) + + cmd = args.pop(0) + if not args: + print help_all() + sys.exit() + + first = args.pop(0) + if first == 'help': + action = help_all + params = [] + if args: + params.append(args.pop(0)) + action = help_controller + if args: + params.append(args.pop(0)) + action = help_method + print action(*params) + sys.exit(0) + + controller = first + if not args: + print help_controller(controller) + sys.exit() + + method = args.pop(0) + params = {} + for x in args: + key, value = args.split('=', 1) + params[key] = value + + pprint.pprint(do_request(controller, method, params)) diff --git a/nova/api/easy.py b/nova/api/easy.py index 0e4f8a8923ef..7468e3115b5b 100644 --- a/nova/api/easy.py +++ b/nova/api/easy.py @@ -25,7 +25,7 @@ The general flow of a request is: (/controller/method) - Parameters are parsed from the request and passed to a method on the controller as keyword arguments. - - Optionally json_body is decoded to provide all the parameters. + - Optionally 'json' is decoded to provide all the parameters. - Actual work is done and a result is returned. - That result is turned into json and returned. @@ -94,7 +94,7 @@ class SundayMorning(wsgi.Router): def __init__(self, mapper=None): if mapper is None: mapper = routes.Mapper() - + self._load_registered_routes(mapper) super(SundayMorning, self).__init__(mapper=mapper) @@ -103,14 +103,18 @@ class SundayMorning(wsgi.Router): mapper.connect('/%s/{action}' % route, controller=ServiceWrapper(EASY_ROUTES[route])) - + class Reflection(object): + """Reflection methods to list available methods.""" def __init__(self): self._methods = {} + self._controllers = {} def _gather_methods(self): methods = {} + controllers = {} for route, handler in EASY_ROUTES.iteritems(): + controllers[route] = handler.__doc__.split('\n')[0] for k in dir(handler): if k.startswith('_'): continue @@ -120,40 +124,63 @@ class Reflection(object): # bunch of ugly formatting stuff argspec = inspect.getargspec(f) - args = [x for x in argspec[0] if x != 'self' and x != 'context'] + args = [x for x in argspec[0] + if x != 'self' and x != 'context'] defaults = argspec[3] and argspec[3] or [] args_r = list(reversed(args)) defaults_r = list(reversed(defaults)) + args_out = [] while args_r: if defaults_r: - args_out.append((args_r.pop(0), defaults_r.pop(0))) + args_out.append((args_r.pop(0), + repr(defaults_r.pop(0)))) else: - args_out.append(str(args_r.pop(0))) + args_out.append((str(args_r.pop(0)),)) + + # if the method accepts keywords + if argspec[2]: + args_out.insert(0, ('**%s' % argspec[2],)) methods['/%s/%s' % (route, k)] = { + 'short_doc': f.__doc__.split('\n')[0], + 'doc': f.__doc__, 'name': k, 'args': list(reversed(args_out))} - return methods + + self._methods = methods + self._controllers = controllers + + def get_controllers(self, context): + """List available controllers.""" + if not self._controllers: + self._gather_methods() + + return self._controllers def get_methods(self, context): + """List available methods.""" if not self._methods: - self._methods = self._gather_methods() + self._gather_methods() method_list = self._methods.keys() method_list.sort() - return {'methods': method_list} + methods = {} + for k in method_list: + methods[k] = self._methods[k]['short_doc'] + return methods def get_method_info(self, context, method): + """Get detailed information about a method.""" if not self._methods: - self._methods = self._gather_methods() + self._gather_methods() return self._methods[method] class ServiceWrapper(wsgi.Controller): def __init__(self, service_handle): self.service_handle = service_handle - + @webob.dec.wsgify def __call__(self, req): arg_dict = req.environ['wsgiorg.routing_args'][1] @@ -165,10 +192,10 @@ class ServiceWrapper(wsgi.Controller): params = {} if 'openstack.params' in req.environ: params = req.environ['openstack.params'] - + # TODO(termie): do some basic normalization on methods method = getattr(self.service_handle, action) - + result = method(context, **params) if type(result) is dict or type(result) is list: return self._serialize(result, req) @@ -181,7 +208,7 @@ class Proxy(object): def __init__(self, app, prefix=None): self.app = app self.prefix = prefix - + def __do_request(self, path, context, **kwargs): req = webob.Request.blank(path) req.method = 'POST' @@ -196,7 +223,7 @@ class Proxy(object): def __getattr__(self, key): if self.prefix is None: return self.__class__(self.app, prefix=key) - + def _wrapper(context, **kwargs): return self.__do_request('/%s/%s' % (self.prefix, key), context, diff --git a/nova/compute/api.py b/nova/compute/api.py index 5f18539a3cbb..005ed7a68fcf 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -40,6 +40,7 @@ def id_to_default_hostname(internal_id): """Default function to generate a hostname given an instance reference.""" return str(internal_id) + def id_to_ec2_hostname(internal_id): digits = [] while internal_id != 0: @@ -47,9 +48,11 @@ def id_to_ec2_hostname(internal_id): digits.append('0123456789abcdefghijklmnopqrstuvwxyz'[remainder]) return "i-%s" % ''.join(reversed(digits)) + HOSTNAME_FORMATTERS = {'default': id_to_default_hostname, 'ec2': id_to_ec2_hostname} + class ComputeAPI(base.Base): """API for interacting with the compute manager.""" @@ -63,6 +66,7 @@ class ComputeAPI(base.Base): super(ComputeAPI, self).__init__(**kwargs) def get_network_topic(self, context, instance_id): + """Get the network topic for an instance.""" try: instance = self.db.instance_get_by_internal_id(context, instance_id) @@ -221,6 +225,7 @@ class ComputeAPI(base.Base): return self.db.instance_update(context, instance_id, kwargs) def delete_instance(self, context, instance_id): + """Terminate and remove an instance.""" logging.debug("Going to try and terminate %d" % instance_id) try: instance = self.db.instance_get_by_internal_id(context, @@ -264,6 +269,7 @@ class ComputeAPI(base.Base): return self.db.instance_get_all(context) def get_instance(self, context, instance_id): + """Get information about a specific instance.""" rv = self.db.instance_get_by_internal_id(context, instance_id) return dict(rv.iteritems()) diff --git a/nova/tests/easy_unittest.py b/nova/tests/easy_unittest.py index 81990d842e2e..cd13c771022f 100644 --- a/nova/tests/easy_unittest.py +++ b/nova/tests/easy_unittest.py @@ -31,6 +31,7 @@ from nova.api import easy from nova.compute import api as compute_api from nova.tests import cloud_unittest + class FakeService(object): def echo(self, context, data): return {'data': data} @@ -49,7 +50,7 @@ class EasyTestCase(test.TestCase): easy.SundayMorning())) self.auth_router = easy.DelegatedAuthMiddleware(self.router) self.context = context.RequestContext('user1', 'proj1') - + def tearDown(self): easy.EASY_ROUTES = {} @@ -61,7 +62,7 @@ class EasyTestCase(test.TestCase): data = json.loads(resp.body) self.assertEqual(data['user'], 'user1') self.assertEqual(data['project'], 'proj1') - + def test_json_params(self): req = webob.Request.blank('/fake/echo') req.environ['openstack.context'] = self.context @@ -99,4 +100,3 @@ class EasyCloudTestCase(cloud_unittest.CloudTestCase): def tearDown(self): super(EasyCloudTestCase, self).tearDown() easy.EASY_ROUTES = {} - diff --git a/nova/utils.py b/nova/utils.py index 7a98ffa5a044..337924f10041 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -386,7 +386,7 @@ def dumps(value): return json.dumps(value) except TypeError: pass - + return json.dumps(to_primitive(value)) diff --git a/nova/wsgi.py b/nova/wsgi.py index c40f043f929f..564805ae7eda 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -105,8 +105,7 @@ class Application(object): class Middleware(Application): """Base WSGI middleware. - - Modelled after Django's middleware this class allows you to + These classes require an application to be initialized that will be called next. By default the middleware will simply call its wrapped app, or you can override __call__ to customize its From b50433d77207c542ee63b7858eb465bb51ba56ea Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 31 Dec 2010 05:37:30 -0600 Subject: [PATCH 12/84] Before merge with xenstore-plugin code --- nova/compute/manager.py | 19 ++- nova/virt/xenapi/vmops.py | 290 +++++++++++++++++++++++++++++++------- nova/virt/xenapi_conn.py | 18 ++- 3 files changed, 267 insertions(+), 60 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index c46587adcc36..4d8c5e1a5367 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -36,6 +36,7 @@ terminating it. import datetime import logging +import string from nova import exception from nova import flags @@ -239,15 +240,21 @@ class ComputeManager(manager.Manager): power_state.RUNNING) logging.debug('instance %s: resetting root password', - instance_ref['name']) - self.db.instance_set_state(context, - instance_id, - power_state.NOSTATE, - 'resetting_password') + instance_ref['name']) + self.db.instance_set_state(context, instance_id, + power_state.NOSTATE, 'resetting_password') #### TODO: (dabo) not sure how we will implement this yet. - self.driver.reset_root_password(instance_ref) + new_pass = self._generate_password(12) + self.driver.reset_root_password(instance_ref, new_pass) self._update_state(context, instance_id) + def _generate_password(self, length=20): + """Generate a random sequence of letters and digits + to be used as a password. + """ + chrs = string.letters + string.digits + return "".join([random.choice(chrs) for i in xrange(length)]) + @exception.wrap_exception def rescue_instance(self, context, instance_id): """Rescue an instance on this server.""" diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3f8f0da69a89..304ff7232261 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -20,7 +20,10 @@ Management class for VM-related functions (spawn, reboot, etc). import json import logging +import os import random +import subprocess +import tempfile import uuid from nova import db @@ -138,14 +141,38 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.clean_reboot', vm) self._session.wait_for_task(instance.id, task) - def reset_root_password(self, instance): - """Reset the root/admin password on the VM instance""" - self.add_to_param_xenstore(instance, "reset_root_password", "requested") - self.add_to_param_xenstore(instance, "TEST", "OMG!") - import time - self.add_to_param_xenstore(instance, "timestamp", time.ctime()) - - + def reset_root_password(self, instance, new_pass): + """Reset the root/admin password on the VM instance. This is + done via an agent running on the VM. Communication between + nova and the agent is done via writing xenstore records. Since + communication is done over the XenAPI RPC calls, we need to + encrypt the password. We're using a simple Diffie-Hellman class + instead of the more advanced one in M2Crypto for compatibility + with the agent code. + """ + # Need to uniquely identify this request. + transaction_id = str(uuid.uuid4()) + # The simple Diffie-Hellman class is used to manage key exchange. + dh = SimpleDH() + args = {'id': transaction_id, 'pub': str(dh.get_public())} + resp = self._make_agent_call('key_init', instance, '', args) + resp_dict = json.loads(resp) + # Successful return code from key_init is 'D0' + if resp_dict['returncode'] != 'D0': + # There was some sort of error + raise RuntimeError(resp_dict['message']) + agent_pub = int(resp_dict['message']) + dh.compute_shared(agent_pub) + enc_pass = dh.encrypt(new_pass) + # Send the encrypted password + args['enc_pass'] = enc_pass + resp = self._make_agent_call('password', instance, '', args) + resp_dict = json.loads(resp) + # Successful return code from password is '0' + if resp_dict['returncode'] != '0': + raise RuntimeError(resp_dict['message']) + return resp_dict['message'] + def destroy(self, instance): """Destroy VM instance""" vm = VMHelper.lookup(self._session, instance.name) @@ -159,7 +186,7 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.hard_shutdown', vm) self._session.wait_for_task(instance.id, task) - except XenAPI.Failure, exc: + except self.XenAPI.Failure, exc: logging.warn(exc) # Disk clean-up if vdis: @@ -167,20 +194,20 @@ class VMOps(object): try: task = self._session.call_xenapi('Async.VDI.destroy', vdi) self._session.wait_for_task(instance.id, task) - except XenAPI.Failure, exc: + except self.XenAPI.Failure, exc: logging.warn(exc) # VM Destroy try: task = self._session.call_xenapi('Async.VM.destroy', vm) self._session.wait_for_task(instance.id, task) - except XenAPI.Failure, exc: + except self.XenAPI.Failure, exc: logging.warn(exc) def _wait_with_callback(self, instance_id, task, callback): ret = None try: ret = self._session.wait_for_task(instance_id, task) - except XenAPI.Failure, exc: + except self.XenAPI.Failure, exc: logging.warn(exc) callback(ret) @@ -235,27 +262,119 @@ class VMOps(object): # TODO: implement this to fix pylint! return 'FAKE CONSOLE OUTPUT of instance' - def dh_keyinit(self, instance): - """Initiates a Diffie-Hellman (or, more precisely, a - Diffie-Hellman-Merkle) key exchange with the agent. It will - compute one side of the exchange and write it to xenstore. - When a response is received, it will then compute the shared - secret key, which is returned. - NOTE: the base and prime are pre-set; this may change in - the future. + def list_from_xenstore(self, vm, path): + """Runs the xenstore-ls command to get a listing of all records + from 'path' downward. Returns a dict with the sub-paths as keys, + and the value stored in those paths as values. If nothing is + found at that path, returns None. """ - base = 5 - prime = 162259276829213363391578010288127 - secret_int = random.randint(100,1000) - val = (base ** secret_int) % prime - msgname = str(uuid.uuid4()) - key = "/data/host/%s" % msgname - self.add_to_param_xenstore(instance, key=key, - value={"name": "keyinit", "value": val}) + ret = self._make_xenstore_call('list_records', vm, path) + try: + return json.loads(ret) + except ValueError: + # Not a valid JSON value + return ret + def read_from_xenstore(self, vm, path): + """Returns the value stored in the xenstore record for the given VM + at the specified location. A XenAPIPlugin.PluginError will be raised + if any error is encountered in the read process. + """ + try: + ret = self._make_xenstore_call('read_record', vm, path, + {'ignore_missing_path': 'True'}) + except self.XenAPI.Failure, e: + print "XENERR", e + return None + except StandardError, e: + print "ERR", type(e), e, e.msg + return None + try: + return json.loads(ret) + except ValueError: + # Not a JSON object + if ret == "None": + # Can't marshall None over RPC calls. + return None + return ret + + def write_to_xenstore(self, vm, path, value): + """Writes the passed value to the xenstore record for the given VM + at the specified location. A XenAPIPlugin.PluginError will be raised + if any error is encountered in the write process. + """ + return self._make_xenstore_call('write_record', vm, path, {'value': json.dumps(value)}) + + def clear_xenstore(self, vm, path): + """Deletes the VM's xenstore record for the specified path. + If there is no such record, the request is ignored. + """ + self._make_xenstore_call('delete_record', vm, path) + + def _make_xenstore_call(self, method, vm, path, addl_args={}): + """Abstracts out the interaction with the xenstore xenapi plugin.""" + return self._make_plugin_call('xenstore.py', method=method, vm=vm, path=path, + addl_args=addl_args) + + def _make_agent_call(self, method, vm, path, addl_args={}): + """Abstracts out the interaction with the agent xenapi plugin.""" + return self._make_plugin_call('agent.py', method=method, vm=vm, path=path, + addl_args=addl_args) + + def _make_plugin_call(self, plugin, method, vm, path, addl_args={}): + vm = self._get_vm_opaque_ref(vm) + rec = self._session.get_xenapi().VM.get_record(vm) + args = {'dom_id': rec['domid'], 'path': path} + args.update(addl_args) + # If the 'testing_mode' attribute is set, add that to the args. + if getattr(self, 'testing_mode', False): + args['testing_mode'] = 'true' + try: + task = self._session.async_call_plugin(plugin, method, args) + ret = self._session.wait_for_task(0, task) + except self.XenAPI.Failure, e: + raise RuntimeError("%s" % e.details[-1]) + return ret + + def add_to_xenstore(self, vm, path, key, value): + """Adds the passed key/value pair to the xenstore record for + the given VM at the specified location. A XenAPIPlugin.PluginError + will be raised if any error is encountered in the write process. + """ + current = self.read_from_xenstore(vm, path) + current[key] = value + self.write_to_xenstore(vm, path, current) + + def remove_from_xenstore(self, vm, path, key_or_keys): + """Takes either a single key or a list of keys and removes + them from the xenstoreirecord data for the given VM. + If the key doesn't exist, the request is ignored. + """ + current = self.list_from_xenstore(vm, path) + if not current: + return + if isinstance(key_or_keys, basestring): + keys = [key_or_keys] + else: + keys = key_or_keys + keys.sort(lambda x,y: cmp(y.count('/'), x.count('/'))) + for key in keys: + if path: + keypath = "%s/%s" % (path, key) + else: + keypath = key + self._make_xenstore_call('delete_record', vm, keypath) + + + ######################################################################## + ###### The following methods interact with the xenstore parameter + ###### record, not the live xenstore. They were created before I + ###### knew the difference, and are left in here in case they prove + ###### to be useful. + ######################################################################## def read_partial_from_param_xenstore(self, instance_or_vm, key_prefix): - """Returns a dict of all the keys in the xenstore for the given instance - that begin with the key_prefix. + """Returns a dict of all the keys in the xenstore parameter record + for the given instance that begin with the key_prefix. """ data = self.read_from_param_xenstore(instance_or_vm) badkeys = [k for k in data.keys() @@ -265,8 +384,8 @@ class VMOps(object): return data def read_from_param_xenstore(self, instance_or_vm, keys=None): - """Returns the xenstore data for the specified VM instance as - a dict. Accepts an optional key or list of keys; if a value for 'keys' + """Returns the xenstore parameter record data for the specified VM instance + as a dict. Accepts an optional key or list of keys; if a value for 'keys' is passed, the returned dict is filtered to only return the values for those keys. """ @@ -286,9 +405,9 @@ class VMOps(object): return ret def add_to_param_xenstore(self, instance_or_vm, key, val): - """Takes a key/value pair and adds it to the xenstore record - for the given vm instance. If the key exists in xenstore, it is - overwritten""" + """Takes a key/value pair and adds it to the xenstore parameter + record for the given vm instance. If the key exists in xenstore, + it is overwritten""" vm = self._get_vm_opaque_ref(instance_or_vm) self.remove_from_param_xenstore(instance_or_vm, key) jsonval = json.dumps(val) @@ -297,27 +416,16 @@ class VMOps(object): def write_to_param_xenstore(self, instance_or_vm, mapping): """Takes a dict and writes each key/value pair to the xenstore - record for the given vm instance. Any existing data for those - keys is overwritten. + parameter record for the given vm instance. Any existing data for + those keys is overwritten. """ for k, v in mapping.iteritems(): self.add_to_param_xenstore(instance_or_vm, k, v) def remove_from_param_xenstore(self, instance_or_vm, key_or_keys): """Takes either a single key or a list of keys and removes - them from the xenstore data for the given VM. If the key - doesn't exist, the request is ignored. - """ - vm = self._get_vm_opaque_ref(instance_or_vm) - if isinstance(key_or_keys, basestring): - keys = [key_or_keys] - else: - keys = key_or_keys - for key in keys: - self._session.call_xenapi_request('VM.remove_from_xenstore_data', (vm, key)) - """Takes either a single key or a list of keys and removes - them from the xenstore data for the given VM. If the key - doesn't exist, the request is ignored. + them from the xenstore parameter record data for the given VM. + If the key doesn't exist, the request is ignored. """ vm = self._get_vm_opaque_ref(instance_or_vm) if isinstance(key_or_keys, basestring): @@ -328,5 +436,85 @@ class VMOps(object): self._session.call_xenapi_request('VM.remove_from_xenstore_data', (vm, key)) def clear_param_xenstore(self, instance_or_vm): - """Removes all data from the xenstore record for this VM.""" + """Removes all data from the xenstore parameter record for this VM.""" self.write_to_param_xenstore(instance_or_vm, {}) + ######################################################################## + + +def _runproc(cmd): + pipe = subprocess.PIPE + return subprocess.Popen([cmd], shell=True, stdin=pipe, stdout=pipe, stderr=pipe, close_fds=True) + +class SimpleDH(object): + """This class wraps all the functionality needed to implement + basic Diffie-Hellman-Merkle key exchange in Python. It features + intelligent defaults for the prime and base numbers needed for the + calculation, while allowing you to supply your own. It requires that + the openssl binary be installed on the system on which this is run, + as it uses that to handle the encryption and decryption. If openssl + is not available, a RuntimeError will be raised. + """ +# def __init__(self, prime=None, base=None): +# """You can specify the values for prime and base if you wish; +# otherwise, reasonable default values will be used. +# """ +# if prime is None: +# self._prime = 162259276829213363391578010288127 +# else: +# self._prime = prime +# if base is None: +# self._base = 5 +# else: +# self._base = base +# self._secret = random.randint(5000, 15000) +# self._shared = self._public = None + def __init__(self, prime=None, base=None, secret=None): + """You can specify the values for prime and base if you wish; + otherwise, reasonable default values will be used. + """ + if prime is None: + self._prime = 162259276829213363391578010288127 + else: + self._prime = prime + if base is None: + self._base = 5 + else: + self._base = base + if secret is None: + self._secret = random.randint(5000, 15000) + else: + self._secret = secret + self._shared = self._public = None + + def get_public(self): + self._public = (self._base ** self._secret) % self._prime + return self._public + + def compute_shared(self, other): + self._shared = (other ** self._secret) % self._prime + return self._shared + + def _run_ssl(self, text, which): + base_cmd = ('cat %(tmpfile)s | openssl enc -aes-128-cbc ' + '-a -pass pass:%(shared)s -nosalt %(dec_flag)s') + if which.lower()[0] == 'd': + dec_flag = ' -d' + else: + dec_flag = '' + fd, tmpfile = tempfile.mkstemp() + os.close(fd) + file(tmpfile, 'w').write(text) + shared = self._shared + cmd = base_cmd % locals() + proc = _runproc(cmd) + proc.wait() + err = proc.stderr.read() + if err: + raise RuntimeError(_('OpenSSL error: %s') % err) + return proc.stdout.read() + + def encrypt(self, text): + return self._run_ssl(text, 'enc') + + def decrypt(self, text): + return self._run_ssl(text, 'dec') diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 7fb7ff15db18..6b65a8c48f66 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -135,9 +135,9 @@ class XenAPIConnection(object): """Reboot VM instance""" self._vmops.reboot(instance) - def reset_root_password(self, instance): + def reset_root_password(self, instance, new_pass): """Reset the root/admin password on the VM instance""" - self._vmops.reset_root_password(instance) + self._vmops.reset_root_password(instance, new_pass) def destroy(self, instance): """Destroy VM instance""" @@ -264,7 +264,19 @@ class XenAPISession(object): status, error_info)) done.send_exception(self.XenAPI.Failure(error_info)) - db.instance_action_create(context.get_admin_context(), action) + + #db.instance_action_create(context.get_admin_context(), action) + import sqlalchemy + from sqlalchemy.exc import IntegrityError as IntegrityError + try: + db.instance_action_create(context.get_admin_context(), action) + except IntegrityError: + # Some methods don't pass unique IDs, so the call to + # instance_action_create() will raise IntegrityErrors. Rather + # than bomb out, I'm explicitly silencing them so that the + # code can continue to work until they fix that method. + pass + except self.XenAPI.Failure, exc: logging.warn(exc) done.send_exception(*sys.exc_info()) From 4f77545cb1ae58484669028fbddb06592b1ee7e4 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 31 Dec 2010 08:54:10 -0600 Subject: [PATCH 13/84] fixed pep8 issues --- nova/virt/xenapi/vmops.py | 64 ++++++++++++++++++--------------------- nova/virt/xenapi_conn.py | 20 ++++++------ 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b0269d960e61..49857e3d90cc 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -61,12 +61,13 @@ class VMOps(object): """Create VM instance""" vm = VMHelper.lookup(self._session, instance.name) if vm is not None: - raise exception.Duplicate(_('Attempted to create non-unique name %s') - % instance.name) + msg = _('Attempted to create non-unique name %s') % instance.name) + raise exception.Duplicate( bridge = db.network_get_by_instance(context.get_admin_context(), instance['id'])['bridge'] - network_ref = NetworkHelper.find_network_with_bridge(self._session, bridge) + network_ref = NetworkHelper.find_network_with_bridge(self._session, + bridge) user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) @@ -222,7 +223,7 @@ class VMOps(object): if resp_dict['returncode'] != '0': raise RuntimeError(resp_dict['message']) return resp_dict['message'] - + def destroy(self, instance): """Destroy VM instance""" vm = VMHelper.lookup(self._session, instance.name) @@ -321,7 +322,7 @@ class VMOps(object): def list_from_xenstore(self, vm, path): """Runs the xenstore-ls command to get a listing of all records from 'path' downward. Returns a dict with the sub-paths as keys, - and the value stored in those paths as values. If nothing is + and the value stored in those paths as values. If nothing is found at that path, returns None. """ ret = self._make_xenstore_call('list_records', vm, path) @@ -355,7 +356,8 @@ class VMOps(object): at the specified location. A XenAPIPlugin.PluginError will be raised if any error is encountered in the write process. """ - return self._make_xenstore_call('write_record', vm, path, {'value': json.dumps(value)}) + return self._make_xenstore_call('write_record', vm, path, + {'value': json.dumps(value)}) def clear_xenstore(self, vm, path): """Deletes the VM's xenstore record for the specified path. @@ -365,13 +367,13 @@ class VMOps(object): def _make_xenstore_call(self, method, vm, path, addl_args={}): """Handles calls to the xenstore xenapi plugin.""" - return self._make_plugin_call('xenstore.py', method=method, vm=vm, path=path, - addl_args=addl_args) + return self._make_plugin_call('xenstore.py', method=method, vm=vm, + path=path, addl_args=addl_args) def _make_agent_call(self, method, vm, path, addl_args={}): """Abstracts out the interaction with the agent xenapi plugin.""" - return self._make_plugin_call('agent.py', method=method, vm=vm, path=path, - addl_args=addl_args) + return self._make_plugin_call('agent.py', method=method, vm=vm, + path=path, addl_args=addl_args) def _make_plugin_call(self, plugin, method, vm, path, addl_args={}): """Abstracts out the process of calling a method of a xenapi plugin. @@ -416,7 +418,7 @@ class VMOps(object): keys = [key_or_keys] else: keys = key_or_keys - keys.sort(lambda x,y: cmp(y.count('/'), x.count('/'))) + keys.sort(lambda x, y: cmp(y.count('/'), x.count('/'))) for key in keys: if path: keypath = "%s/%s" % (path, key) @@ -424,9 +426,8 @@ class VMOps(object): keypath = key self._make_xenstore_call('delete_record', vm, keypath) - ######################################################################## - ###### The following methods interact with the xenstore parameter + ###### The following methods interact with the xenstore parameter ###### record, not the live xenstore. They were created before I ###### knew the difference, and are left in here in case they prove ###### to be useful. They all have '_param' added to their method @@ -444,13 +445,14 @@ class VMOps(object): return data def read_from_param_xenstore(self, instance_or_vm, keys=None): - """Returns the xenstore parameter record data for the specified VM instance - as a dict. Accepts an optional key or list of keys; if a value for 'keys' - is passed, the returned dict is filtered to only return the values - for those keys. + """Returns the xenstore parameter record data for the specified VM + instance as a dict. Accepts an optional key or list of keys; if a + value for 'keys' is passed, the returned dict is filtered to only + return the values for those keys. """ vm = self._get_vm_opaque_ref(instance_or_vm) - data = self._session.call_xenapi_request('VM.get_xenstore_data', (vm, )) + data = self._session.call_xenapi_request('VM.get_xenstore_data', + (vm, )) ret = {} if keys is None: keys = data.keys() @@ -493,7 +495,8 @@ class VMOps(object): else: keys = key_or_keys for key in keys: - self._session.call_xenapi_request('VM.remove_from_xenstore_data', (vm, key)) + self._session.call_xenapi_request('VM.remove_from_xenstore_data', + (vm, key)) def clear_param_xenstore(self, instance_or_vm): """Removes all data from the xenstore parameter record for this VM.""" @@ -503,7 +506,9 @@ class VMOps(object): def _runproc(cmd): pipe = subprocess.PIPE - return subprocess.Popen([cmd], shell=True, stdin=pipe, stdout=pipe, stderr=pipe, close_fds=True) + return subprocess.Popen([cmd], shell=True, stdin=pipe, stdout=pipe, + stderr=pipe, close_fds=True) + class SimpleDH(object): """This class wraps all the functionality needed to implement @@ -514,23 +519,12 @@ class SimpleDH(object): as it uses that to handle the encryption and decryption. If openssl is not available, a RuntimeError will be raised. """ -# def __init__(self, prime=None, base=None): -# """You can specify the values for prime and base if you wish; -# otherwise, reasonable default values will be used. -# """ -# if prime is None: -# self._prime = 162259276829213363391578010288127 -# else: -# self._prime = prime -# if base is None: -# self._base = 5 -# else: -# self._base = base -# self._secret = random.randint(5000, 15000) -# self._shared = self._public = None def __init__(self, prime=None, base=None, secret=None): """You can specify the values for prime and base if you wish; - otherwise, reasonable default values will be used. + otherwise, reasonable default values will be used. You may also + specify the integer value for 'secret', but this should only be + done while testing when you need reproducible values. Otherwise, + any security benefits are lost. """ if prime is None: self._prime = 162259276829213363391578010288127 diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index fe302c450a08..3baae6188b6e 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -275,16 +275,16 @@ class XenAPISession(object): error_info)) done.send_exception(self.XenAPI.Failure(error_info)) db.instance_action_create(context.get_admin_context(), action) -# import sqlalchemy -# from sqlalchemy.exc import IntegrityError as IntegrityError -# try: -# db.instance_action_create(context.get_admin_context(), action) -# except IntegrityError: -# # Some methods don't pass unique IDs, so the call to -# # instance_action_create() will raise IntegrityErrors. Rather -# # than bomb out, I'm explicitly silencing them so that the -# # code can continue to work until they fix that method. -# pass +# import sqlalchemy +# from sqlalchemy.exc import IntegrityError as IntegrityError +# try: +# db.instance_action_create(context.get_admin_context(), action) +# except IntegrityError: +# # Some methods don't pass unique IDs, so the call to +# # instance_action_create() will raise IntegrityErrors. Rather +# # than bomb out, I'm explicitly silencing them so that the +# # code can continue to work until they fix that method. +# pass except self.XenAPI.Failure, exc: logging.warn(exc) From 9c17d68bb670f389a16e05d4306ad0a720e7a1e4 Mon Sep 17 00:00:00 2001 From: MORITA Kazutaka Date: Sat, 1 Jan 2011 02:53:45 +0900 Subject: [PATCH 14/84] Add support for various block device types (block, network, file) --- nova/virt/libvirt_conn.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 00edfbdc8ddd..51c805c2f420 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -223,11 +223,26 @@ class LibvirtConnection(object): def attach_volume(self, instance_name, device_path, mountpoint): virt_dom = self._conn.lookupByName(instance_name) mount_device = mountpoint.rpartition("/")[2] - xml = """ - - - - """ % (device_path, mount_device) + xml = '' + (protocol, vol_name) = device_path.split(':') + if device_path.startswith('/dev/'): + xml = """ + + + + """ % (device_path, mount_device) + elif vol_name != '': + xml = """ + + + + """ % (protocol, vol_name, mount_device) + else: + xml = """ + + + + """ % (device_path, mount_device) virt_dom.attachDevice(xml) def _get_disk_xml(self, xml, device): From f8272fc0b38be55d383860b6138e79a8a74965be Mon Sep 17 00:00:00 2001 From: MORITA Kazutaka Date: Sat, 1 Jan 2011 02:53:51 +0900 Subject: [PATCH 15/84] Add support for Sheepdog volumes --- nova/volume/driver.py | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 8353b9712a3a..2e8a3c816298 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -312,3 +312,52 @@ class FakeISCSIDriver(ISCSIDriver): """Execute that simply logs the command.""" logging.debug(_("FAKE ISCSI: %s"), cmd) return (None, None) + + +class SheepdogDriver(VolumeDriver): + """Executes commands relating to Sheepdog Volumes""" + + def check_for_setup_error(self): + """Returns an error if prerequisites aren't met""" + try: + (out, err) = self._execute("collie cluster info") + if not out.startswith('running'): + raise exception.Error(_("Sheepdog is not working: %s") % out) + except exception.ProcessExecutionError: + raise exception.Error(_("Sheepdog is not working")) + + def create_volume(self, volume): + """Creates a sheepdog volume""" + if int(volume['size']) == 0: + sizestr = '100M' + else: + sizestr = '%sG' % volume['size'] + self._try_execute("qemu-img create sheepdog:%s %s" % + (volume['name'], sizestr)) + + def delete_volume(self, volume): + """Deletes a logical volume""" + self._try_execute("collie vdi delete %s" % volume['name']) + + def local_path(self, volume): + return "sheepdog:%s" % volume['name'] + + def ensure_export(self, context, volume): + """Safely and synchronously recreates an export for a logical volume""" + pass + + def create_export(self, context, volume): + """Exports the volume""" + pass + + def remove_export(self, context, volume): + """Removes an export for a logical volume""" + pass + + def discover_volume(self, volume): + """Discover volume on a remote host""" + return "sheepdog:%s" % volume['name'] + + def undiscover_volume(self, volume): + """Undiscover volume on a remote host""" + pass From 108352d5c132f6accc79974d8c646a2bc7d4f127 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 31 Dec 2010 12:21:04 -0600 Subject: [PATCH 16/84] Updated the password generation code --- nova/compute/manager.py | 8 +++++--- nova/virt/fake.py | 2 +- nova/virt/xenapi/vmops.py | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 8ebd87f28ae3..583fabe42af5 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -36,6 +36,7 @@ terminating it. import datetime import logging +import random import string from nova import exception @@ -247,7 +248,7 @@ class ComputeManager(manager.Manager): self.driver.snapshot(instance_ref, name) @exception.wrap_exception - def reset_root_password(self, context, instance_id): + def reset_root_password(self, context, instance_id, new_pass=None): """Reset the root/admin password for an instance on this server.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) @@ -264,8 +265,9 @@ class ComputeManager(manager.Manager): instance_ref['name']) self.db.instance_set_state(context, instance_id, power_state.NOSTATE, 'resetting_password') - #### TODO: (dabo) not sure how we will implement this yet. - new_pass = self._generate_password(12) + if new_pass is None: + # Generate a random, 12-character password + new_pass = self._generate_password(12) self.driver.reset_root_password(instance_ref, new_pass) self._update_state(context, instance_id) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 13c67c4ee462..fdc8ac5fb1fa 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -138,7 +138,7 @@ class FakeConnection(object): """ pass - def reset_root_password(self, instance): + def reset_root_password(self, instance, new_pass): """ Reset the root password on the specified instance. diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 49857e3d90cc..4ce8d819bfb9 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -61,8 +61,8 @@ class VMOps(object): """Create VM instance""" vm = VMHelper.lookup(self._session, instance.name) if vm is not None: - msg = _('Attempted to create non-unique name %s') % instance.name) - raise exception.Duplicate( + msg = _('Attempted to create non-unique name %s') % instance.name + raise exception.Duplicate(msg) bridge = db.network_get_by_instance(context.get_admin_context(), instance['id'])['bridge'] From 56969837fdf1a9e5316443ce72b32ae268ed2947 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Tue, 4 Jan 2011 04:21:50 -0500 Subject: [PATCH 17/84] Fixed conflict with r515 --- nova/virt/libvirt.uml.xml.template | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 nova/virt/libvirt.uml.xml.template diff --git a/nova/virt/libvirt.uml.xml.template b/nova/virt/libvirt.uml.xml.template deleted file mode 100644 index 0d355b81cf9b..000000000000 --- a/nova/virt/libvirt.uml.xml.template +++ /dev/null @@ -1,27 +0,0 @@ - - %(name)s - %(memory_kb)s - - %(type)s - /usr/bin/linux - /dev/ubda1 - - - - - - - - - - - - - - - - - - - - From 96facdeee025cdf33df0b16abeeeb97f9ec87e70 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Tue, 4 Jan 2011 04:27:39 -0500 Subject: [PATCH 18/84] Fixed for pep8 --- nova/virt/libvirt_conn.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9a99b1a51f9e..c656931d6afb 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -550,7 +550,7 @@ class LibvirtConnection(object): "\n" "\n") % (net, mask,net_v6,mask_v6) + "value=\"%s\" />\n") % (net, mask, net_v6, mask_v6) else: extra_params = "\n" @@ -777,7 +777,6 @@ class NWFilterFirewall(object): ''' - nova_ra_filter = ''' d707fa71-4fb5-4b27-9ab7-ba5ca19c8804 ''' - nova_vpn_filter = ''' 2086015e-cf03-11df-8c5d-080027c27973 @@ -795,7 +793,6 @@ class NWFilterFirewall(object): ''' - def nova_base_ipv4_filter(self): retval = "" for protocol in ['tcp', 'udp', 'icmp']: @@ -832,8 +829,9 @@ class NWFilterFirewall(object): def nova_project_filter_v6(self): retval = "" % project for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: - retval += """ - <%s srcipaddr='$PROJNETV6' srcipmask='$PROJMASKV6' /> + retval += """ + <%s srcipaddr='$PROJNETV6' + srcipmask='$PROJMASKV6' /> """ % (protocol) retval += '' return retval @@ -872,7 +870,7 @@ class NWFilterFirewall(object): if FLAGS.allow_project_net_traffic: nwfilter_xml += " \n" if(FLAGS.use_ipv6): - nwfilter_xml += " \n" + nwfilter_xml += " \n" for security_group in instance.security_groups: self.ensure_security_group_filter(security_group['id']) @@ -892,7 +890,7 @@ class NWFilterFirewall(object): security_group_id) rule_xml = "" version = 4 - v6protocol = {'tcp':'tcp-ipv6', 'udp':'udp-ipv6', 'icmp':'icmpv6'} + v6protocol = {'tcp': 'tcp-ipv6', 'udp': 'udp-ipv6', 'icmp': 'icmpv6'} for rule in security_group.rules: rule_xml += "" if rule.cidr: @@ -904,7 +902,6 @@ class NWFilterFirewall(object): else: rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ (rule.protocol, net, mask) - if rule.protocol in ['tcp', 'udp']: rule_xml += "dstportstart='%s' dstportend='%s' " % \ (rule.from_port, rule.to_port) From 505becef0704cc801f957d2931c8b994e2df92ca Mon Sep 17 00:00:00 2001 From: nova Date: Tue, 4 Jan 2011 05:00:21 -0500 Subject: [PATCH 19/84] Fixed bug --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index c656931d6afb..de7f6341fad2 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -827,7 +827,7 @@ class NWFilterFirewall(object): return retval def nova_project_filter_v6(self): - retval = "" % project + retval = "" for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: retval += """ <%s srcipaddr='$PROJNETV6' From c528be81a5d0acaea5077c183ec4d15356d457d5 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Tue, 4 Jan 2011 05:35:13 -0500 Subject: [PATCH 20/84] Fixed bug in libvirt --- nova/virt/libvirt_conn.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index de7f6341fad2..f955441885f6 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -114,6 +114,9 @@ def _get_net_and_mask(cidr): net = IPy.IP(cidr) return str(net.net()), str(net.netmask()) +def _get_ip_version(cidr): + net = IPy.IP(cidr) + return int(net.version()) class LibvirtConnection(object): @@ -484,7 +487,8 @@ class LibvirtConnection(object): 'netmask': network_ref['netmask'], 'gateway': network_ref['gateway'], 'broadcast': network_ref['broadcast'], - 'dns': network_ref['dns']} + 'dns': network_ref['dns'], + 'ra_server': network_ref['ra_server']} if key or net: if key: logging.info(_('instance %s: injecting key into image %s'), @@ -541,8 +545,8 @@ class LibvirtConnection(object): if FLAGS.allow_project_net_traffic: net, mask = _get_net_and_mask(network['cidr']) - net_v6, mask_v6 = self._get_net_and_mask( - network_ref['cidr_v6']) + net_v6, mask_v6 = _get_net_and_mask( + network['cidr_v6']) extra_params = ("\n" " Date: Tue, 4 Jan 2011 07:40:29 -0500 Subject: [PATCH 21/84] Some Bug Fix --- nova/utils.py | 4 +--- nova/virt/libvirt.xml.template | 2 +- nova/virt/libvirt_conn.py | 16 +++++++++++----- smoketests/user_smoketests.py | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 0eab4b1528e4..c5e2f751764d 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -211,8 +211,6 @@ def get_my_ip(): def get_my_linklocal(interface): - if getattr(FLAGS, 'fake_tests', None): - return 'fe00::' try: if_str = execute("ifconfig %s" % interface) condition = "\s+inet6\s+addr:\s+([0-9a-f:]+/\d+)\s+Scope:Link" @@ -224,7 +222,7 @@ def get_my_linklocal(interface): return None except RuntimeError as ex: logging.warn("Couldn't get Link Local IP of %s :%s", interface, ex) - return None + return 'fe00::' def to_global_ipv6(prefix, mac): diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index f0f56fc62fdb..52c95ee574ab 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -66,7 +66,7 @@ - + #if $getVar('extra_params', False) ${extra_params} #end if diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index f955441885f6..521ac97fcccd 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -114,6 +114,10 @@ def _get_net_and_mask(cidr): net = IPy.IP(cidr) return str(net.net()), str(net.netmask()) +def _get_net_and_prefixlen(cidr): + net = IPy.IP(cidr) + return str(net.net()), str(net.prefixlen()) + def _get_ip_version(cidr): net = IPy.IP(cidr) return int(net.version()) @@ -354,6 +358,7 @@ class LibvirtConnection(object): power_state.NOSTATE, 'launching') NWFilterFirewall(self._conn).setup_nwfilters_for_instance(instance) + self._create_image(instance, xml) self._conn.createXML(xml, 0) logging.debug(_("instance %s: is running"), instance['name']) @@ -545,7 +550,7 @@ class LibvirtConnection(object): if FLAGS.allow_project_net_traffic: net, mask = _get_net_and_mask(network['cidr']) - net_v6, mask_v6 = _get_net_and_mask( + net_v6, prefixlen_v6 = _get_net_and_prefixlen( network['cidr_v6']) extra_params = ("\n" @@ -554,7 +559,7 @@ class LibvirtConnection(object): "\n" "\n") % (net, mask, net_v6, mask_v6) + "value=\"%s\" />\n") % (net, mask, net_v6, prefixlen_v6) else: extra_params = "\n" @@ -882,7 +887,7 @@ class NWFilterFirewall(object): nwfilter_xml += (" \n") % security_group['id'] nwfilter_xml += "" - + logging.debug(nwfilter_xml) self._define_filter(nwfilter_xml) def ensure_security_group_filter(self, security_group_id): @@ -899,11 +904,12 @@ class NWFilterFirewall(object): rule_xml += "" if rule.cidr: version = _get_ip_version(rule.cidr) - net, mask = _get_net_and_mask(rule.cidr) if(FLAGS.use_ipv6 and version == 6): + net, prefixlen = _get_net_and_prefixlen(rule.cidr) rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ - (v6protocol[rrule.protocol], net, mask) + (v6protocol[rrule.protocol], net, prefixlen) else: + net, mask = _get_net_and_mask(rule.cidr) rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ (rule.protocol, net, mask) if rule.protocol in ['tcp', 'udp']: diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index 8bb2ddebb2a8..f4ee30542239 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -193,7 +193,7 @@ class InstanceTests(UserSmokeTestCase): else: break else: - self.fail('could not ssh to instance') + self.fail('could not ssh to instance v6') def test_006_can_allocate_elastic_ip(self): result = self.conn.allocate_address() From 35d3050511ef513ff440fbd9f8b44695ea8be797 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 4 Jan 2011 14:07:46 -0800 Subject: [PATCH 22/84] rename Easy API to Direct API --- bin/{nova-easy-api => nova-direct-api} | 20 +++++----- nova/api/{easy.py => direct.py} | 38 +++++++++---------- .../{easy_unittest.py => test_direct.py} | 36 +++++++++--------- 3 files changed, 47 insertions(+), 47 deletions(-) rename bin/{nova-easy-api => nova-direct-api} (75%) rename nova/api/{easy.py => direct.py} (94%) rename nova/tests/{easy_unittest.py => test_direct.py} (77%) diff --git a/bin/nova-easy-api b/bin/nova-direct-api similarity index 75% rename from bin/nova-easy-api rename to bin/nova-direct-api index e8e86b4fb41b..43046e6c2ea9 100755 --- a/bin/nova-easy-api +++ b/bin/nova-direct-api @@ -37,25 +37,25 @@ gettext.install('nova', unicode=1) from nova import flags from nova import utils from nova import wsgi -from nova.api import easy +from nova.api import direct from nova.compute import api as compute_api FLAGS = flags.FLAGS -flags.DEFINE_integer('easy_port', 8001, 'Easy API port') -flags.DEFINE_string('easy_host', '0.0.0.0', 'Easy API host') +flags.DEFINE_integer('direct_port', 8001, 'Direct API port') +flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host') if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) - easy.register_service('compute', compute_api.ComputeAPI()) - easy.register_service('reflect', easy.Reflection()) - router = easy.SundayMorning() - with_json = easy.JsonParamsMiddleware(router) - with_req = easy.ReqParamsMiddleware(with_json) - with_auth = easy.DelegatedAuthMiddleware(with_req) + direct.register_service('compute', compute_api.ComputeAPI()) + direct.register_service('reflect', direct.Reflection()) + router = direct.Router() + with_json = direct.JsonParamsMiddleware(router) + with_req = direct.PostParamsMiddleware(with_json) + with_auth = direct.DelegatedAuthMiddleware(with_req) server = wsgi.Server() - server.start(with_auth, FLAGS.easy_port, host=FLAGS.easy_host) + server.start(with_auth, FLAGS.direct_port, host=FLAGS.direct_host) server.wait() diff --git a/nova/api/easy.py b/nova/api/direct.py similarity index 94% rename from nova/api/easy.py rename to nova/api/direct.py index 7468e3115b5b..81b3ae202622 100644 --- a/nova/api/easy.py +++ b/nova/api/direct.py @@ -43,11 +43,25 @@ from nova import utils from nova import wsgi -EASY_ROUTES = {} +ROUTES = {} def register_service(path, handle): - EASY_ROUTES[path] = handle + ROUTES[path] = handle + + +class Router(wsgi.Router): + def __init__(self, mapper=None): + if mapper is None: + mapper = routes.Mapper() + + self._load_registered_routes(mapper) + super(Router, self).__init__(mapper=mapper) + + def _load_registered_routes(self, mapper): + for route in ROUTES: + mapper.connect('/%s/{action}' % route, + controller=ServiceWrapper(ROUTES[route])) class DelegatedAuthMiddleware(wsgi.Middleware): @@ -76,7 +90,7 @@ class JsonParamsMiddleware(wsgi.Middleware): request.environ['openstack.params'] = params -class ReqParamsMiddleware(wsgi.Middleware): +class PostParamsMiddleware(wsgi.Middleware): def process_request(self, request): params_parsed = request.params params = {} @@ -90,20 +104,6 @@ class ReqParamsMiddleware(wsgi.Middleware): request.environ['openstack.params'] = params -class SundayMorning(wsgi.Router): - def __init__(self, mapper=None): - if mapper is None: - mapper = routes.Mapper() - - self._load_registered_routes(mapper) - super(SundayMorning, self).__init__(mapper=mapper) - - def _load_registered_routes(self, mapper): - for route in EASY_ROUTES: - mapper.connect('/%s/{action}' % route, - controller=ServiceWrapper(EASY_ROUTES[route])) - - class Reflection(object): """Reflection methods to list available methods.""" def __init__(self): @@ -113,7 +113,7 @@ class Reflection(object): def _gather_methods(self): methods = {} controllers = {} - for route, handler in EASY_ROUTES.iteritems(): + for route, handler in ROUTES.iteritems(): controllers[route] = handler.__doc__.split('\n')[0] for k in dir(handler): if k.startswith('_'): @@ -204,7 +204,7 @@ class ServiceWrapper(wsgi.Controller): class Proxy(object): - """Pretend an Easy API endpoint is an object.""" + """Pretend a Direct API endpoint is an object.""" def __init__(self, app, prefix=None): self.app = app self.prefix = prefix diff --git a/nova/tests/easy_unittest.py b/nova/tests/test_direct.py similarity index 77% rename from nova/tests/easy_unittest.py rename to nova/tests/test_direct.py index cd13c771022f..d73c64ce038f 100644 --- a/nova/tests/easy_unittest.py +++ b/nova/tests/test_direct.py @@ -16,7 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Tests for Easy API.""" +"""Tests for Direct API.""" import json import logging @@ -27,7 +27,7 @@ from nova import context from nova import exception from nova import test from nova import utils -from nova.api import easy +from nova.api import direct from nova.compute import api as compute_api from nova.tests import cloud_unittest @@ -41,18 +41,18 @@ class FakeService(object): 'project': context.project_id} -class EasyTestCase(test.TestCase): +class DirectTestCase(test.TestCase): def setUp(self): - super(EasyTestCase, self).setUp() - easy.register_service('fake', FakeService()) - self.router = easy.ReqParamsMiddleware( - easy.JsonParamsMiddleware( - easy.SundayMorning())) - self.auth_router = easy.DelegatedAuthMiddleware(self.router) + super(DirectTestCase, self).setUp() + direct.register_service('fake', FakeService()) + self.router = direct.PostParamsMiddleware( + direct.JsonParamsMiddleware( + direct.Router())) + self.auth_router = direct.DelegatedAuthMiddleware(self.router) self.context = context.RequestContext('user1', 'proj1') def tearDown(self): - easy.EASY_ROUTES = {} + direct.ROUTES = {} def test_delegated_auth(self): req = webob.Request.blank('/fake/context') @@ -82,21 +82,21 @@ class EasyTestCase(test.TestCase): self.assertEqual(resp_parsed['data'], 'foo') def test_proxy(self): - proxy = easy.Proxy(self.router) + proxy = direct.Proxy(self.router) rv = proxy.fake.echo(self.context, data='baz') self.assertEqual(rv['data'], 'baz') -class EasyCloudTestCase(cloud_unittest.CloudTestCase): +class DirectCloudTestCase(cloud_unittest.CloudTestCase): def setUp(self): - super(EasyCloudTestCase, self).setUp() + super(DirectCloudTestCase, self).setUp() compute_handle = compute_api.ComputeAPI(self.cloud.network_manager, self.cloud.image_service) - easy.register_service('compute', compute_handle) - self.router = easy.JsonParamsMiddleware(easy.SundayMorning()) - proxy = easy.Proxy(self.router) + direct.register_service('compute', compute_handle) + self.router = direct.JsonParamsMiddleware(direct.Router()) + proxy = direct.Proxy(self.router) self.cloud.compute_api = proxy.compute def tearDown(self): - super(EasyCloudTestCase, self).tearDown() - easy.EASY_ROUTES = {} + super(DirectCloudTestCase, self).tearDown() + direct.ROUTES = {} From 1a66771aaf49d16d4131a1a787a1fda39aa680fd Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 4 Jan 2011 14:37:04 -0800 Subject: [PATCH 23/84] fix typo in stack tool --- bin/stack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/stack b/bin/stack index 284dbf4fc6f3..42feacd63834 100755 --- a/bin/stack +++ b/bin/stack @@ -139,7 +139,7 @@ if __name__ == '__main__': method = args.pop(0) params = {} for x in args: - key, value = args.split('=', 1) + key, value = x.split('=', 1) params[key] = value pprint.pprint(do_request(controller, method, params)) From 5679caa48b90ecebf9a1143bf92cec0e7c0ed1f8 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 4 Jan 2011 14:39:48 -0800 Subject: [PATCH 24/84] rename easy to direct in the scripts --- bin/nova-direct-api | 2 +- bin/stack | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 43046e6c2ea9..e7dd14fb27b8 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -18,7 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Starter script for Nova Easy API.""" +"""Starter script for Nova Direct API.""" import gettext import os diff --git a/bin/stack b/bin/stack index 42feacd63834..7a6ce5960d1c 100755 --- a/bin/stack +++ b/bin/stack @@ -17,7 +17,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""CLI for the Easy API.""" +"""CLI for the Direct API.""" import eventlet eventlet.monkey_patch() @@ -42,10 +42,10 @@ from nova import utils FLAGS = gflags.FLAGS -gflags.DEFINE_string('host', '127.0.0.1', 'Easy API host') -gflags.DEFINE_integer('port', 8001, 'Easy API host') -gflags.DEFINE_string('user', 'user1', 'Easy API username') -gflags.DEFINE_string('project', 'proj1', 'Easy API project') +gflags.DEFINE_string('host', '127.0.0.1', 'Direct API host') +gflags.DEFINE_integer('port', 8001, 'Direct API host') +gflags.DEFINE_string('user', 'user1', 'Direct API username') +gflags.DEFINE_string('project', 'proj1', 'Direct API project') USAGE = """usage: stack [options] [arg1=value arg2=value] From 1cc5e933ccc29a88d09d2050e5224ee27eda767c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 5 Jan 2011 08:05:11 +0000 Subject: [PATCH 25/84] stop using partitions and first pass at cow images --- nova/compute/disk.py | 108 +++++++++++++++++++++------------ nova/virt/libvirt.xml.template | 6 ++ nova/virt/libvirt_conn.py | 106 ++++++++++++++++++-------------- 3 files changed, 137 insertions(+), 83 deletions(-) diff --git a/nova/compute/disk.py b/nova/compute/disk.py index 814a258cd61e..766f27d3523c 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -28,6 +28,7 @@ import tempfile from nova import exception from nova import flags +from nova import utils FLAGS = flags.FLAGS @@ -37,8 +38,7 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, 'block_size to use for dd') -def partition(infile, outfile, local_bytes=0, resize=True, - local_type='ext2', execute=None): +def partition(infile, outfile, local_bytes=0, resize=True, local_type='ext2'): """ Turns a partition (infile) into a bootable drive image (outfile). @@ -61,10 +61,10 @@ def partition(infile, outfile, local_bytes=0, resize=True, file_size = os.path.getsize(infile) if resize and file_size < FLAGS.minimum_root_size: last_sector = FLAGS.minimum_root_size / sector_size - 1 - execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + utils.execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' % (infile, last_sector, sector_size)) - execute('e2fsck -fp %s' % infile, check_exit_code=False) - execute('resize2fs %s' % infile) + utils.execute('e2fsck -fp %s' % infile, check_exit_code=False) + utils.execute('resize2fs %s' % infile) file_size = FLAGS.minimum_root_size elif file_size % sector_size != 0: logging.warn(_("Input partition size not evenly divisible by" @@ -83,37 +83,37 @@ def partition(infile, outfile, local_bytes=0, resize=True, last_sector = local_last # e # create an empty file - execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + utils.execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' % (outfile, mbr_last, sector_size)) # make mbr partition - execute('parted --script %s mklabel msdos' % outfile) + utils.execute('parted --script %s mklabel msdos' % outfile) # append primary file - execute('dd if=%s of=%s bs=%s conv=notrunc,fsync oflag=append' + utils.execute('dd if=%s of=%s bs=%s conv=notrunc,fsync oflag=append' % (infile, outfile, FLAGS.block_size)) # make primary partition - execute('parted --script %s mkpart primary %ds %ds' + utils.execute('parted --script %s mkpart primary %ds %ds' % (outfile, primary_first, primary_last)) if local_bytes > 0: # make the file bigger - execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + utils.execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' % (outfile, last_sector, sector_size)) # make and format local partition - execute('parted --script %s mkpartfs primary %s %ds %ds' + utils.execute('parted --script %s mkpartfs primary %s %ds %ds' % (outfile, local_type, local_first, local_last)) -def extend(image, size, execute): +def extend(image, size): file_size = os.path.getsize(image) if file_size >= size: return - return execute('truncate -s size %s' % (image,)) + return utils.execute('truncate -s size %s' % (image,)) -def inject_data(image, key=None, net=None, partition=None, execute=None): +def inject_data(image, key=None, net=None, partition=None): """Injects a ssh key and optionally net data into a disk image. it will mount the image as a fully partitioned disk and attempt to inject @@ -122,15 +122,11 @@ def inject_data(image, key=None, net=None, partition=None, execute=None): If partition is not specified it mounts the image as a single partition. """ - out, err = execute('sudo losetup --find --show %s' % image) - if err: - raise exception.Error(_('Could not attach image to loopback: %s') - % err) - device = out.strip() + device = _link_device(image) try: if not partition is None: # create partition - out, err = execute('sudo kpartx -a %s' % device) + out, err = utils.execute('sudo kpartx -a %s' % device) if err: raise exception.Error(_('Failed to load partition: %s') % err) mapped_device = '/dev/mapper/%sp%s' % (device.split('/')[-1], @@ -146,12 +142,12 @@ def inject_data(image, key=None, net=None, partition=None, execute=None): mapped_device) # Configure ext2fs so that it doesn't auto-check every N boots - out, err = execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) + out, err = utils.execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) tmpdir = tempfile.mkdtemp() try: # mount loopback to dir - out, err = execute( + out, err = utils.execute( 'sudo mount %s %s' % (mapped_device, tmpdir)) if err: raise exception.Error(_('Failed to mount filesystem: %s') @@ -160,45 +156,79 @@ def inject_data(image, key=None, net=None, partition=None, execute=None): try: if key: # inject key file - _inject_key_into_fs(key, tmpdir, execute=execute) + _inject_key_into_fs(key, tmpdir) if net: - _inject_net_into_fs(net, tmpdir, execute=execute) + _inject_net_into_fs(net, tmpdir) finally: # unmount device - execute('sudo umount %s' % mapped_device) + utils.execute('sudo umount %s' % mapped_device) finally: # remove temporary directory - execute('rmdir %s' % tmpdir) + utils.execute('rmdir %s' % tmpdir) if not partition is None: # remove partitions - execute('sudo kpartx -d %s' % device) + utils.execute('sudo kpartx -d %s' % device) finally: - # remove loopback - execute('sudo losetup --detach %s' % device) + _unlink_device(image, device) -def _inject_key_into_fs(key, fs, execute=None): +def _link_device(image): + if FLAGS.use_cow_images: + device = _allocate_device() + utils.execute('sudo qemu-nbd --connect=%s %s' % (device, image)) + else: + out, err = utils.execute('sudo losetup --find --show %s' % image) + if err: + raise exception.Error(_('Could not attach image to loopback: %s') + % err) + return out.strip() + + +def _unlink_device(image, device): + if FLAGS.use_cow_images: + utils.execute('sudo qemu-nbd --disconnect %s' % image) + _free_device(device) + else: + utils.execute('sudo losetup --detach %s' % device) + + +_DEVICES = ['/dev/nbd%s' % i for i in xrange(16)] + +def _allocate_device(): + # NOTE(vish): This assumes no other processes are using nbd devices. + # It will race cause a race condition if multiple + # workers are running on a given machine. + if not _DEVICES: + raise exception.Error(_('No free nbd devices')) + return _DEVICES.pop() + + +def _free_device(device): + _DEVICES.append(device) + + +def _inject_key_into_fs(key, fs): """Add the given public ssh key to root's authorized_keys. key is an ssh key string. fs is the path to the base of the filesystem into which to inject the key. """ sshdir = os.path.join(fs, 'root', '.ssh') - execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter - execute('sudo chown root %s' % sshdir) - execute('sudo chmod 700 %s' % sshdir) + utils.execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter + utils.execute('sudo chown root %s' % sshdir) + utils.execute('sudo chmod 700 %s' % sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') + utils.execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') -def _inject_net_into_fs(net, fs, execute=None): +def _inject_net_into_fs(net, fs): """Inject /etc/network/interfaces into the filesystem rooted at fs. net is the contents of /etc/network/interfaces. """ netdir = os.path.join(os.path.join(fs, 'etc'), 'network') - execute('sudo mkdir -p %s' % netdir) # existing dir doesn't matter - execute('sudo chown root:root %s' % netdir) - execute('sudo chmod 755 %s' % netdir) + utils.execute('sudo mkdir -p %s' % netdir) # existing dir doesn't matter + utils.execute('sudo chown root:root %s' % netdir) + utils.execute('sudo chmod 755 %s' % netdir) netfile = os.path.join(netdir, 'interfaces') - execute('sudo tee %s' % netfile, net) + utils.execute('sudo tee %s' % netfile, net) diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index 3fb2243dacbd..d6d9d3de62dd 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -58,6 +58,12 @@ + #if $getVar('local', False) + + + + + #end if #end if diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 00edfbdc8ddd..883913926ae1 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -85,6 +85,9 @@ flags.DEFINE_string('libvirt_uri', flags.DEFINE_bool('allow_project_net_traffic', True, 'Whether to allow in project network traffic') +flags.DEFINE_bool('use_cow_images', + True, + 'Whether to use cow images') def get_connection(read_only): @@ -418,19 +421,50 @@ class LibvirtConnection(object): return self._dump_file(fpath) + def _get_image(self, image_id, target, user, project, size=None): + if not os.path.exists(target): + if FLAGS.use_cow_images: + base = os.path.join(FLAGS.instances_path, '_base') + if not os.path.exists(base): + images.fetch(image_id, base, user, project) + if size: + # TODO(vish): Attempt to resize the filesystem + disk.extend(base, size) + utils.execute('qemu-img create -f qcow2 -o ' + 'cluster_size=2M,backing_file=%s %s' + % (base, target)) + else: + images.fetch(image_id, target, user, project) + if size: + # TODO(vish): Attempt to resize the filesystem + disk.extend(target, size) + + def _get_local(self, local_gb, target): + if not os.path.exists(target): + last_mb = local_gb * 1024 - 1 + if FLAGS.use_cow_images: + base = os.path.join(FLAGS.instances_path, '_base') + if not os.path.exists(base): + utils.execute('dd if=/dev/zero of=%s bs=1M count=1' + 'seek=%s' % (base, last_mb)) + utils.execute('qemu-img create -f qcow2 -o ' + 'cluster_size=2M,backing_file=%s %s' + % (base, target)) + else: + utils.execute('dd if=/dev/zero of=%s bs=1M count=1' + 'seek=%s' % (base, last_mb)) + def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): # syntactic nicety - basepath = lambda fname = '', prefix = prefix: os.path.join( - FLAGS.instances_path, - inst['name'], - prefix + fname) + def basepath(fname='', prefix=prefix): + return os.path.join(FLAGS.instances_path, + inst['name'], + prefix + fname) # ensure directories exist and are writable utils.execute('mkdir -p %s' % basepath(prefix='')) utils.execute('chmod 0777 %s' % basepath(prefix='')) - # TODO(termie): these are blocking calls, it would be great - # if they weren't. logging.info(_('instance %s: Creating image'), inst['name']) f = open(basepath('libvirt.xml'), 'w') f.write(libvirt_xml) @@ -447,23 +481,26 @@ class LibvirtConnection(object): disk_images = {'image_id': inst['image_id'], 'kernel_id': inst['kernel_id'], 'ramdisk_id': inst['ramdisk_id']} - if not os.path.exists(basepath('disk')): - images.fetch(inst.image_id, basepath('disk-raw'), user, - project) - if inst['kernel_id']: - if not os.path.exists(basepath('kernel')): - images.fetch(inst['kernel_id'], basepath('kernel'), - user, project) - if inst['ramdisk_id']: - if not os.path.exists(basepath('ramdisk')): - images.fetch(inst['ramdisk_id'], basepath('ramdisk'), - user, project) + if disk_images['kernel_id']: + self._get_image(disk_images['kernel_id'], basepath('kernel'), + user, project) + if disk_images['ramdisk_id']: + self._get_image(disk_images['ramdisk_id'], basepath('ramdisk'), + user, project) - def execute(cmd, process_input=None, check_exit_code=True): - return utils.execute(cmd=cmd, - process_input=process_input, - check_exit_code=check_exit_code) + + size = FLAGS.minimum_root_size + if not FLAGS.use_cow_images: + if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': + size = None + + self._get_image(disk_images['image_id'], basepath('disk'), + user, project, size) + + type_data = instance_types.INSTANCE_TYPES[inst['instance_type']] + self._get_local(type_data['local_gb'], basepath('local'), + user, project, size) # For now, we assume that if we're not using a kernel, we're using a # partitioned disk image where the target partition is the first @@ -493,34 +530,14 @@ class LibvirtConnection(object): logging.info(_('instance %s: injecting net into image %s'), inst['name'], inst.image_id) try: - disk.inject_data(basepath('disk-raw'), key, net, - partition=target_partition, - execute=execute) + disk.inject_data(basepath('disk'), key, net, + partition=target_partition) except Exception as e: # This could be a windows image, or a vmdk format disk logging.warn(_('instance %s: ignoring error injecting data' ' into image %s (%s)'), inst['name'], inst.image_id, e) - if inst['kernel_id']: - if os.path.exists(basepath('disk')): - utils.execute('rm -f %s' % basepath('disk')) - - local_bytes = (instance_types.INSTANCE_TYPES[inst.instance_type] - ['local_gb'] - * 1024 * 1024 * 1024) - - resize = True - if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': - resize = False - - if inst['kernel_id']: - disk.partition(basepath('disk-raw'), basepath('disk'), - local_bytes, resize, execute=execute) - else: - os.rename(basepath('disk-raw'), basepath('disk')) - disk.extend(basepath('disk'), local_bytes, execute=execute) - if FLAGS.libvirt_type == 'uml': utils.execute('sudo chown root %s' % basepath('disk')) @@ -558,7 +575,8 @@ class LibvirtConnection(object): 'ip_address': ip_address, 'dhcp_server': dhcp_server, 'extra_params': extra_params, - 'rescue': rescue} + 'rescue': rescue, + 'local': instance_type['local_gb']} if not rescue: if instance['kernel_id']: xml_info['kernel'] = xml_info['basepath'] + "/kernel" From b14a8975dece3be18216cf2ad790be0c8fd22f7a Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 5 Jan 2011 09:52:45 -0500 Subject: [PATCH 26/84] The _update method in base Instance class overides dns_name_v6,so fixed it. --- contrib/boto_v6/ec2/instance.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/boto_v6/ec2/instance.py b/contrib/boto_v6/ec2/instance.py index 255114935e42..18b5cd33ad35 100644 --- a/contrib/boto_v6/ec2/instance.py +++ b/contrib/boto_v6/ec2/instance.py @@ -25,9 +25,13 @@ class ReservationV6(Reservation): class InstanceV6(Instance): def __init__(self, connection=None): Instance.__init__(self, connection) - self.public_dns_name_v6 = None + self.dns_name_v6 = None def endElement(self, name, value, connection): Instance.endElement(self, name, value, connection) if name == 'dnsNameV6': self.dns_name_v6 = value + + def _update(self, updated): + self.__dict__.update(updated.__dict__) + self.dns_name_v6 = updated.dns_name_v6 From 28bf4e2df324db79a81a853d39cb5912985c2e45 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 5 Jan 2011 10:25:16 -0500 Subject: [PATCH 27/84] Fixed bug in nova_project_filter_v6 --- nova/virt/libvirt_conn.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 521ac97fcccd..a592f5d6b20a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -114,14 +114,17 @@ def _get_net_and_mask(cidr): net = IPy.IP(cidr) return str(net.net()), str(net.netmask()) + def _get_net_and_prefixlen(cidr): net = IPy.IP(cidr) return str(net.net()), str(net.prefixlen()) + def _get_ip_version(cidr): net = IPy.IP(cidr) return int(net.version()) + class LibvirtConnection(object): def __init__(self, read_only): @@ -559,7 +562,8 @@ class LibvirtConnection(object): "\n" "\n") % (net, mask, net_v6, prefixlen_v6) + "value=\"%s\" />\n") % \ + (net, mask, net_v6, prefixlen_v6) else: extra_params = "\n" @@ -838,7 +842,8 @@ class NWFilterFirewall(object): def nova_project_filter_v6(self): retval = "" for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: - retval += """ + retval += """ <%s srcipaddr='$PROJNETV6' srcipmask='$PROJMASKV6' /> """ % (protocol) From b47f37d0f9c06f2c4bc5adcf3afcececa2354324 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 5 Jan 2011 10:35:15 -0500 Subject: [PATCH 28/84] Fixed misspelled variable --- nova/virt/libvirt_conn.py | 2 +- smoketests/user_smoketests.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index a592f5d6b20a..1f3c69f653fc 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -912,7 +912,7 @@ class NWFilterFirewall(object): if(FLAGS.use_ipv6 and version == 6): net, prefixlen = _get_net_and_prefixlen(rule.cidr) rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ - (v6protocol[rrule.protocol], net, prefixlen) + (v6protocol[rule.protocol], net, prefixlen) else: net, mask = _get_net_and_mask(rule.cidr) rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index f4ee30542239..181acdfe598a 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -162,13 +162,18 @@ class InstanceTests(UserSmokeTestCase): 'ping -c1 %s' % self.data['private_ip']) if status == 0: break - if FLAGS.use_ipv6: + else: + self.fail('could not ping instance') + + if FLAGS.use_ipv6: + for x in xrange(120): + # ping waits for 1 second status, output = commands.getstatusoutput( 'ping6 -c1 %s' % self.data['ip_v6']) if status == 0: break - else: - self.fail('could not ping instance') + else: + self.fail('could not ping instance') def test_005_can_ssh_to_private_ip(self): for x in xrange(30): From 69b7a0d69c3ac79b84c2bda19d379606c5a323ab Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 5 Jan 2011 10:42:15 -0500 Subject: [PATCH 29/84] Removed debug message which is not needed. --- nova/virt/libvirt_conn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 1f3c69f653fc..4fba164a9fd1 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -892,7 +892,6 @@ class NWFilterFirewall(object): nwfilter_xml += (" \n") % security_group['id'] nwfilter_xml += "" - logging.debug(nwfilter_xml) self._define_filter(nwfilter_xml) def ensure_security_group_filter(self, security_group_id): From 40b156f74e90a94abb255950f29d714f4bc4c428 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 5 Jan 2011 12:36:47 -0500 Subject: [PATCH 30/84] Fixed:Create instance fails when use_ipv6=False --- nova/virt/libvirt_conn.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 4fba164a9fd1..8197342dfc6d 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -490,13 +490,16 @@ class LibvirtConnection(object): if network_ref['injected']: admin_context = context.get_admin_context() address = db.instance_get_fixed_address(admin_context, inst['id']) + ra_server = network_ref['ra_server'] + if not ra_server: + ra_server = "fd00::" with open(FLAGS.injected_network_template) as f: net = f.read() % {'address': address, 'netmask': network_ref['netmask'], 'gateway': network_ref['gateway'], 'broadcast': network_ref['broadcast'], 'dns': network_ref['dns'], - 'ra_server': network_ref['ra_server']} + 'ra_server': ra_server} if key or net: if key: logging.info(_('instance %s: injecting key into image %s'), @@ -550,12 +553,14 @@ class LibvirtConnection(object): # Assume that the gateway also acts as the dhcp server. dhcp_server = network['gateway'] ra_server = network['ra_server'] - + if not ra_server: + ra_server = 'fd00::' if FLAGS.allow_project_net_traffic: - net, mask = _get_net_and_mask(network['cidr']) - net_v6, prefixlen_v6 = _get_net_and_prefixlen( + if FLAGS.use_ipv6: + net, mask = _get_net_and_mask(network['cidr']) + net_v6, prefixlen_v6 = _get_net_and_prefixlen( network['cidr_v6']) - extra_params = ("\n" "\n" @@ -564,6 +569,13 @@ class LibvirtConnection(object): "\n") % \ (net, mask, net_v6, prefixlen_v6) + else: + net, mask = _get_net_and_mask(network['cidr']) + extra_params = ("\n" + "\n") % \ + (net, mask) else: extra_params = "\n" @@ -860,12 +872,14 @@ class NWFilterFirewall(object): self._define_filter(self.nova_base_ipv4_filter) self._define_filter(self.nova_base_ipv6_filter) self._define_filter(self.nova_dhcp_filter) - self._define_filter(self.nova_ra_filter) + if FLAGS.use_ipv6: + self._define_filter(self.nova_ra_filter) self._define_filter(self.nova_base_filter) self._define_filter(self.nova_vpn_filter) if FLAGS.allow_project_net_traffic: self._define_filter(self.nova_project_filter) - self._define_filter(self.nova_project_filter_v6) + if FLAGS.use_ipv6: + self._define_filter(self.nova_project_filter_v6) def setup_nwfilters_for_instance(self, instance): """ From 90ced5f211c3a53389d2f5d7413f9289770b279a Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 5 Jan 2011 12:38:34 -0500 Subject: [PATCH 31/84] Fixed for pep8 --- nova/virt/libvirt_conn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 8197342dfc6d..b19988822c54 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -490,9 +490,9 @@ class LibvirtConnection(object): if network_ref['injected']: admin_context = context.get_admin_context() address = db.instance_get_fixed_address(admin_context, inst['id']) - ra_server = network_ref['ra_server'] + ra_server = network_ref['ra_server'] if not ra_server: - ra_server = "fd00::" + ra_server = "fd00::" with open(FLAGS.injected_network_template) as f: net = f.read() % {'address': address, 'netmask': network_ref['netmask'], @@ -554,7 +554,7 @@ class LibvirtConnection(object): dhcp_server = network['gateway'] ra_server = network['ra_server'] if not ra_server: - ra_server = 'fd00::' + ra_server = 'fd00::' if FLAGS.allow_project_net_traffic: if FLAGS.use_ipv6: net, mask = _get_net_and_mask(network['cidr']) From f85eba86b04253612e2272b3eb6a9fd79fab6567 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 5 Jan 2011 12:39:35 -0500 Subject: [PATCH 32/84] missing _() --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/utils.py b/nova/utils.py index c5e2f751764d..afe7422d9e54 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -221,7 +221,7 @@ def get_my_linklocal(interface): else: return None except RuntimeError as ex: - logging.warn("Couldn't get Link Local IP of %s :%s", interface, ex) + logging.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex) return 'fe00::' From b5f8ab0e913c121a80ff0efe358960099e7c87f8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 5 Jan 2011 19:16:17 +0000 Subject: [PATCH 33/84] fix injection and xml --- contrib/nova.sh | 5 +- nova/compute/disk.py | 9 +-- nova/virt/libvirt.xml.template | 10 ++- nova/virt/libvirt_conn.py | 107 ++++++++++++++++++++++----------- 4 files changed, 88 insertions(+), 43 deletions(-) diff --git a/contrib/nova.sh b/contrib/nova.sh index da1ba030c3f1..80bf6789b98b 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -82,9 +82,10 @@ if [ "$CMD" == "install" ]; then sudo /etc/init.d/iscsitarget restart sudo modprobe kvm sudo /etc/init.d/libvirt-bin restart + sudo modprobe nbd sudo apt-get install -y python-twisted python-sqlalchemy python-mox python-greenlet python-carrot - sudo apt-get install -y python-daemon python-eventlet python-gflags python-tornado python-ipy - sudo apt-get install -y python-libvirt python-libxml2 python-routes + sudo apt-get install -y python-daemon python-eventlet python-gflags python-ipy + sudo apt-get install -y python-libvirt python-libxml2 python-routes python-cheetah if [ "$USE_MYSQL" == 1 ]; then cat <= size: return - return utils.execute('truncate -s size %s' % (image,)) + return utils.execute('truncate -s %s %s' % (size, image)) def inject_data(image, key=None, net=None, partition=None): @@ -175,7 +175,8 @@ def inject_data(image, key=None, net=None, partition=None): def _link_device(image): if FLAGS.use_cow_images: device = _allocate_device() - utils.execute('sudo qemu-nbd --connect=%s %s' % (device, image)) + utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) + return device else: out, err = utils.execute('sudo losetup --find --show %s' % image) if err: @@ -184,9 +185,9 @@ def _link_device(image): return out.strip() -def _unlink_device(image, device): +def _unlink_device(device): if FLAGS.use_cow_images: - utils.execute('sudo qemu-nbd --disconnect %s' % image) + utils.execute('sudo qemu-nbd -d %s' % device) _free_device(device) else: utils.execute('sudo losetup --detach %s' % device) diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index d6d9d3de62dd..995d8f4698f2 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -7,13 +7,13 @@ #set $disk_bus = 'uml' uml /usr/bin/linux - /dev/ubda1 + /dev/ubda #else #if $type == 'xen' #set $disk_prefix = 'sd' #set $disk_bus = 'scsi' linux - /dev/xvda1 + /dev/xvda #else #set $disk_prefix = 'vd' #set $disk_bus = 'virtio' @@ -28,7 +28,7 @@ #if $type == 'xen' ro #else - root=/dev/vda1 console=ttyS0 + root=/dev/vda console=ttyS0 #end if #if $getVar('ramdisk', None) ${ramdisk} @@ -46,20 +46,24 @@ #if $getVar('rescue', False) + + #else + #if $getVar('local', False) + diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 883913926ae1..ae725b766a74 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -118,6 +118,32 @@ def _get_net_and_mask(cidr): return str(net.net()), str(net.netmask()) +def wrap_cow_image(key): + def _decorator(retrieve_fn): + def _wrap(*args, **kwargs): + target = kwargs['target'] + if not os.path.exists(target): + if FLAGS.use_cow_images: + base_dir = os.path.join(FLAGS.instances_path, '_base') + if not os.path.exists(base_dir): + os.mkdir(base_dir) + os.chmod(base_dir, 0777) + base = os.path.join(base_dir, str(kwargs[key])) + if not os.path.exists(base): + kwargs['target'] = base + retrieve_fn(*args, **kwargs) + if kwargs.get('cow'): + utils.execute('qemu-img create -f qcow2 -o ' + 'cluster_size=2M,backing_file=%s %s' + % (base, target)) + else: + utils.execute('cp %s %s' % (base, target)) + else: + retrieve_fn(*args, **kwargs) + _wrap.func_name = retrieve_fn.func_name + return _wrap + return _decorator + class LibvirtConnection(object): def __init__(self, read_only): @@ -421,38 +447,36 @@ class LibvirtConnection(object): return self._dump_file(fpath) - def _get_image(self, image_id, target, user, project, size=None): + def _get_image(self, retrieve_fn, key, target, *args, **kwargs): if not os.path.exists(target): if FLAGS.use_cow_images: - base = os.path.join(FLAGS.instances_path, '_base') + base_dir = os.path.join(FLAGS.instances_path, '_base') + if not os.path.exists(base_dir): + os.mkdir(base_dir) + os.chmod(base_dir, 0777) + base = os.path.join(base_dir, kwargs[key]) if not os.path.exists(base): - images.fetch(image_id, base, user, project) - if size: - # TODO(vish): Attempt to resize the filesystem - disk.extend(base, size) + retrieve_fn(target=base, *args, **kwargs) utils.execute('qemu-img create -f qcow2 -o ' 'cluster_size=2M,backing_file=%s %s' % (base, target)) else: - images.fetch(image_id, target, user, project) - if size: - # TODO(vish): Attempt to resize the filesystem - disk.extend(target, size) + retrieve_fn(target=base, *args, **kwargs) - def _get_local(self, local_gb, target): - if not os.path.exists(target): - last_mb = local_gb * 1024 - 1 - if FLAGS.use_cow_images: - base = os.path.join(FLAGS.instances_path, '_base') - if not os.path.exists(base): - utils.execute('dd if=/dev/zero of=%s bs=1M count=1' - 'seek=%s' % (base, last_mb)) - utils.execute('qemu-img create -f qcow2 -o ' - 'cluster_size=2M,backing_file=%s %s' - % (base, target)) - else: - utils.execute('dd if=/dev/zero of=%s bs=1M count=1' - 'seek=%s' % (base, last_mb)) + @wrap_cow_image('image_id') + def _fetch_image(self, target, image_id, user, project, + size=None, **kwargs): + images.fetch(image_id, target, user, project) + # TODO(vish): resize filesystem + if size: + disk.extend(target, size) + + @wrap_cow_image('local_gb') + def _create_local(self, target, local_gb): + last_mb = local_gb * 1024 - 1 + utils.execute('dd if=/dev/zero of=%s bs=1M count=1' + 'seek=%s' % (target, last_mb)) + # TODO(vish): format disk def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): # syntactic nicety @@ -483,11 +507,15 @@ class LibvirtConnection(object): 'ramdisk_id': inst['ramdisk_id']} if disk_images['kernel_id']: - self._get_image(disk_images['kernel_id'], basepath('kernel'), - user, project) + self._fetch_image(target=basepath('kernel'), + image_id=disk_images['kernel_id'], + user=user, + project=project) if disk_images['ramdisk_id']: - self._get_image(disk_images['ramdisk_id'], basepath('ramdisk'), - user, project) + self._fetch_image(target=basepath('ramdisk'), + image_id=disk_images['ramdisk_id'], + user=user, + project=project) size = FLAGS.minimum_root_size @@ -495,12 +523,17 @@ class LibvirtConnection(object): if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': size = None - self._get_image(disk_images['image_id'], basepath('disk'), - user, project, size) - + self._fetch_image(target=basepath('disk'), + image_id=disk_images['image_id'], + user=user, + project=project, + size=size, + cow=True) type_data = instance_types.INSTANCE_TYPES[inst['instance_type']] - self._get_local(type_data['local_gb'], basepath('local'), - user, project, size) + + if type_data['local_gb']: + self._create_local(target=basepath('local'), + local_gb=type_data['local_gb']) # For now, we assume that if we're not using a kernel, we're using a # partitioned disk image where the target partition is the first @@ -534,6 +567,7 @@ class LibvirtConnection(object): partition=target_partition) except Exception as e: # This could be a windows image, or a vmdk format disk + logging.exception('test') logging.warn(_('instance %s: ignoring error injecting data' ' into image %s (%s)'), inst['name'], inst.image_id, e) @@ -563,6 +597,10 @@ class LibvirtConnection(object): "value=\"%s\" />\n") % (net, mask) else: extra_params = "\n" + if FLAGS.use_cow_images: + driver_type = 'qcow2' + else: + driver_type = 'raw' xml_info = {'type': FLAGS.libvirt_type, 'name': instance['name'], @@ -576,7 +614,8 @@ class LibvirtConnection(object): 'dhcp_server': dhcp_server, 'extra_params': extra_params, 'rescue': rescue, - 'local': instance_type['local_gb']} + 'local': instance_type['local_gb'], + 'driver_type': driver_type} if not rescue: if instance['kernel_id']: xml_info['kernel'] = xml_info['basepath'] + "/kernel" From f1f292a787ba20134c007da087bd9585d1875e86 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 5 Jan 2011 19:50:39 +0000 Subject: [PATCH 34/84] more fixes, docstrings --- nova/compute/disk.py | 18 +++++---- nova/virt/libvirt_conn.py | 83 +++++++++++++++++++-------------------- 2 files changed, 51 insertions(+), 50 deletions(-) diff --git a/nova/compute/disk.py b/nova/compute/disk.py index f640bdbbb141..ac0d689d597b 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -107,13 +107,15 @@ def partition(infile, outfile, local_bytes=0, resize=True, local_type='ext2'): def extend(image, size): + """Increase image to size""" file_size = os.path.getsize(image) if file_size >= size: return + # TODO(vish): attempt to resize filesystem return utils.execute('truncate -s %s %s' % (size, image)) -def inject_data(image, key=None, net=None, partition=None): +def inject_data(image, key=None, net=None, partition=None, nbd=False): """Injects a ssh key and optionally net data into a disk image. it will mount the image as a fully partitioned disk and attempt to inject @@ -122,7 +124,7 @@ def inject_data(image, key=None, net=None, partition=None): If partition is not specified it mounts the image as a single partition. """ - device = _link_device(image) + device = _link_device(image, nbd) try: if not partition is None: # create partition @@ -169,11 +171,12 @@ def inject_data(image, key=None, net=None, partition=None): # remove partitions utils.execute('sudo kpartx -d %s' % device) finally: - _unlink_device(image, device) + _unlink_device(device, nbd) -def _link_device(image): - if FLAGS.use_cow_images: +def _link_device(image, nbd): + """Link image to device using loopback or nbd""" + if nbd: device = _allocate_device() utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) return device @@ -185,8 +188,9 @@ def _link_device(image): return out.strip() -def _unlink_device(device): - if FLAGS.use_cow_images: +def _unlink_device(device, nbd): + """Unlink image from device using loopback or nbd""" + if nbd: utils.execute('sudo qemu-nbd -d %s' % device) _free_device(device) else: diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index ae725b766a74..95a374603ed0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -118,28 +118,38 @@ def _get_net_and_mask(cidr): return str(net.net()), str(net.netmask()) -def wrap_cow_image(key): +def wrap_image_cache(key): + """Decorator for a method that creates an image to store it in cache. + + This wrapper will save the image into a common store and create a + copy for use by the hypervisor. + + The underlying method should be called with kwargs and specify + a kwarg of target representing where the image will be saved. The + key argument to the wrapper defines which kwarg of the underlying + method to use as the filename of the base image. The filename needs + to be unique to a given image. + + If kwargs['cow'] is True, it will make a CoW image instead of a copy. + """ def _decorator(retrieve_fn): def _wrap(*args, **kwargs): target = kwargs['target'] if not os.path.exists(target): - if FLAGS.use_cow_images: - base_dir = os.path.join(FLAGS.instances_path, '_base') - if not os.path.exists(base_dir): - os.mkdir(base_dir) - os.chmod(base_dir, 0777) - base = os.path.join(base_dir, str(kwargs[key])) - if not os.path.exists(base): - kwargs['target'] = base - retrieve_fn(*args, **kwargs) - if kwargs.get('cow'): - utils.execute('qemu-img create -f qcow2 -o ' - 'cluster_size=2M,backing_file=%s %s' - % (base, target)) - else: - utils.execute('cp %s %s' % (base, target)) - else: + base_dir = os.path.join(FLAGS.instances_path, '_base') + if not os.path.exists(base_dir): + os.mkdir(base_dir) + os.chmod(base_dir, 0777) + base = os.path.join(base_dir, str(kwargs[key])) + if not os.path.exists(base): + kwargs['target'] = base retrieve_fn(*args, **kwargs) + if kwargs.get('cow'): + utils.execute('qemu-img create -f qcow2 -o ' + 'cluster_size=2M,backing_file=%s %s' + % (base, target)) + else: + utils.execute('cp %s %s' % (base, target)) _wrap.func_name = retrieve_fn.func_name return _wrap return _decorator @@ -447,36 +457,21 @@ class LibvirtConnection(object): return self._dump_file(fpath) - def _get_image(self, retrieve_fn, key, target, *args, **kwargs): - if not os.path.exists(target): - if FLAGS.use_cow_images: - base_dir = os.path.join(FLAGS.instances_path, '_base') - if not os.path.exists(base_dir): - os.mkdir(base_dir) - os.chmod(base_dir, 0777) - base = os.path.join(base_dir, kwargs[key]) - if not os.path.exists(base): - retrieve_fn(target=base, *args, **kwargs) - utils.execute('qemu-img create -f qcow2 -o ' - 'cluster_size=2M,backing_file=%s %s' - % (base, target)) - else: - retrieve_fn(target=base, *args, **kwargs) - - @wrap_cow_image('image_id') + @wrap_image_cache('image_id') def _fetch_image(self, target, image_id, user, project, - size=None, **kwargs): + size=None, cow=False): + """Grab image and optionally attempt to resize it""" images.fetch(image_id, target, user, project) - # TODO(vish): resize filesystem if size: disk.extend(target, size) - @wrap_cow_image('local_gb') - def _create_local(self, target, local_gb): + @wrap_image_cache('local_gb') + def _create_local(self, target, local_gb, cow=False): + """Create a blank image of specified size""" last_mb = local_gb * 1024 - 1 - utils.execute('dd if=/dev/zero of=%s bs=1M count=1' + utils.execute('dd if=/dev/zero of=%s bs=1M count=1 ' 'seek=%s' % (target, last_mb)) - # TODO(vish): format disk + # TODO(vish): should we format disk by default? def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): # syntactic nicety @@ -528,12 +523,13 @@ class LibvirtConnection(object): user=user, project=project, size=size, - cow=True) + cow=FLAGS.use_cow_images) type_data = instance_types.INSTANCE_TYPES[inst['instance_type']] if type_data['local_gb']: self._create_local(target=basepath('local'), - local_gb=type_data['local_gb']) + local_gb=type_data['local_gb'], + cow=FLAGS.use_cow_images) # For now, we assume that if we're not using a kernel, we're using a # partitioned disk image where the target partition is the first @@ -564,7 +560,8 @@ class LibvirtConnection(object): inst['name'], inst.image_id) try: disk.inject_data(basepath('disk'), key, net, - partition=target_partition) + partition=target_partition, + nbd=FLAGS.use_cow_images) except Exception as e: # This could be a windows image, or a vmdk format disk logging.exception('test') From 3d30bb1706812c4e6f9c1e01b373bb076a9f7ee3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 5 Jan 2011 19:52:55 +0000 Subject: [PATCH 35/84] pep8 cleanup --- nova/compute/disk.py | 1 + nova/virt/libvirt_conn.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/compute/disk.py b/nova/compute/disk.py index ac0d689d597b..bbcd55678436 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -199,6 +199,7 @@ def _unlink_device(device, nbd): _DEVICES = ['/dev/nbd%s' % i for i in xrange(16)] + def _allocate_device(): # NOTE(vish): This assumes no other processes are using nbd devices. # It will race cause a race condition if multiple diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 95a374603ed0..77ff281d56b9 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -154,6 +154,7 @@ def wrap_image_cache(key): return _wrap return _decorator + class LibvirtConnection(object): def __init__(self, read_only): @@ -512,7 +513,6 @@ class LibvirtConnection(object): user=user, project=project) - size = FLAGS.minimum_root_size if not FLAGS.use_cow_images: if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': From a6b82b3015a64922a0733bd0dd5463b1a49ca080 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 5 Jan 2011 20:46:18 +0000 Subject: [PATCH 36/84] simplify decorator into a wrapper fn --- nova/compute/disk.py | 10 +++- nova/virt/libvirt_conn.py | 104 +++++++++++++++++++------------------- 2 files changed, 59 insertions(+), 55 deletions(-) diff --git a/nova/compute/disk.py b/nova/compute/disk.py index bbcd55678436..8b7453f322fb 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -25,6 +25,7 @@ Includes injection of SSH PGP keys into authorized_keys file. import logging import os import tempfile +import time from nova import exception from nova import flags @@ -111,8 +112,10 @@ def extend(image, size): file_size = os.path.getsize(image) if file_size >= size: return - # TODO(vish): attempt to resize filesystem - return utils.execute('truncate -s %s %s' % (size, image)) + utils.execute('truncate -s %s %s' % (size, image)) + # NOTE(vish): attempts to resize filesystem + utils.execute('e2fsck -fp %s' % image, check_exit_code=False) + utils.execute('resize2fs %s' % image, check_exit_code=False) def inject_data(image, key=None, net=None, partition=None, nbd=False): @@ -179,6 +182,9 @@ def _link_device(image, nbd): if nbd: device = _allocate_device() utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) + # NOTE(vish): this forks into another process, so give it a chance + # to set up before continuuing + time.sleep(1) return device else: out, err = utils.execute('sudo losetup --find --show %s' % image) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 77ff281d56b9..f2803ab5504c 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -118,43 +118,6 @@ def _get_net_and_mask(cidr): return str(net.net()), str(net.netmask()) -def wrap_image_cache(key): - """Decorator for a method that creates an image to store it in cache. - - This wrapper will save the image into a common store and create a - copy for use by the hypervisor. - - The underlying method should be called with kwargs and specify - a kwarg of target representing where the image will be saved. The - key argument to the wrapper defines which kwarg of the underlying - method to use as the filename of the base image. The filename needs - to be unique to a given image. - - If kwargs['cow'] is True, it will make a CoW image instead of a copy. - """ - def _decorator(retrieve_fn): - def _wrap(*args, **kwargs): - target = kwargs['target'] - if not os.path.exists(target): - base_dir = os.path.join(FLAGS.instances_path, '_base') - if not os.path.exists(base_dir): - os.mkdir(base_dir) - os.chmod(base_dir, 0777) - base = os.path.join(base_dir, str(kwargs[key])) - if not os.path.exists(base): - kwargs['target'] = base - retrieve_fn(*args, **kwargs) - if kwargs.get('cow'): - utils.execute('qemu-img create -f qcow2 -o ' - 'cluster_size=2M,backing_file=%s %s' - % (base, target)) - else: - utils.execute('cp %s %s' % (base, target)) - _wrap.func_name = retrieve_fn.func_name - return _wrap - return _decorator - - class LibvirtConnection(object): def __init__(self, read_only): @@ -458,16 +421,42 @@ class LibvirtConnection(object): return self._dump_file(fpath) - @wrap_image_cache('image_id') - def _fetch_image(self, target, image_id, user, project, - size=None, cow=False): + def _cache_image(self, fn, target, fname, cow=False, *args, **kwargs): + """Wrapper to cache a method that creates an image. + + This wrapper will save the image into a common store and create a + copy for use by the hypervisor. + + The underlying method should specify a kwarg of target representing + where the image will be saved. + + fname is used as the filename of the base image. The filename needs + to be fname to a given image. + + If cow is True, it will make a CoW image instead of a copy. + """ + if not os.path.exists(target): + base_dir = os.path.join(FLAGS.instances_path, '_base') + if not os.path.exists(base_dir): + os.mkdir(base_dir) + os.chmod(base_dir, 0777) + base = os.path.join(base_dir, fname) + if not os.path.exists(base): + fn(target=base, *args, **kwargs) + if cow: + utils.execute('qemu-img create -f qcow2 -o ' + 'cluster_size=2M,backing_file=%s %s' + % (base, target)) + else: + utils.execute('cp %s %s' % (base, target)) + + def _fetch_image(self, target, image_id, user, project, size=None): """Grab image and optionally attempt to resize it""" images.fetch(image_id, target, user, project) if size: disk.extend(target, size) - @wrap_image_cache('local_gb') - def _create_local(self, target, local_gb, cow=False): + def _create_local(self, target, local_gb): """Create a blank image of specified size""" last_mb = local_gb * 1024 - 1 utils.execute('dd if=/dev/zero of=%s bs=1M count=1 ' @@ -503,33 +492,42 @@ class LibvirtConnection(object): 'ramdisk_id': inst['ramdisk_id']} if disk_images['kernel_id']: - self._fetch_image(target=basepath('kernel'), + self._cache_image(fn=self._fetch_image, + target=basepath('kernel'), + fname=disk_images['kernel_id'], image_id=disk_images['kernel_id'], user=user, project=project) if disk_images['ramdisk_id']: - self._fetch_image(target=basepath('ramdisk'), + self._cache_image(fn=self._fetch_image, + target=basepath('ramdisk'), + fname=disk_images['ramdisk_id'], image_id=disk_images['ramdisk_id'], user=user, project=project) + root_fname = disk_images['image_id'] size = FLAGS.minimum_root_size - if not FLAGS.use_cow_images: - if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': - size = None + if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': + size = None + root_fname += "_sm" - self._fetch_image(target=basepath('disk'), + self._cache_image(fn=self._fetch_image, + target=basepath('disk'), + fname=root_fname, + cow=FLAGS.use_cow_images, image_id=disk_images['image_id'], user=user, project=project, - size=size, - cow=FLAGS.use_cow_images) + size=size) type_data = instance_types.INSTANCE_TYPES[inst['instance_type']] if type_data['local_gb']: - self._create_local(target=basepath('local'), - local_gb=type_data['local_gb'], - cow=FLAGS.use_cow_images) + self._cache_image(fn=self._create_local, + target=basepath('local'), + fname="local_%s" % type_data['local_gb'], + cow=FLAGS.use_cow_images, + local_gb=type_data['local_gb']) # For now, we assume that if we're not using a kernel, we're using a # partitioned disk image where the target partition is the first From f67802d62ee530b4e81aaf108dfd3813c84550b2 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Wed, 5 Jan 2011 16:41:50 -0600 Subject: [PATCH 37/84] intermediate work --- nova/api/__init__.py | 14 ++++++++++++-- nova/api/openstack/servers.py | 7 ++++--- nova/compute/api.py | 2 +- nova/virt/xenapi/vmops.py | 19 ++++++++++++++----- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 26fed847bb15..ff936bed247f 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -59,13 +59,23 @@ class API(wsgi.Router): mapper.connect("/", controller=self.osapi_versions, conditions=osapi_subdomain) - mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API(), - conditions=osapi_subdomain) + mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API()) mapper.connect("/", controller=self.ec2api_versions, conditions=ec2api_subdomain) mapper.connect("/services/{path_info:.*}", controller=ec2.API(), conditions=ec2api_subdomain) + +# mapper.connect("/", controller=self.osapi_versions, +# conditions=osapi_subdomain) +# mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API(), +# conditions=osapi_subdomain) +# +# mapper.connect("/", controller=self.ec2api_versions, +# conditions=ec2api_subdomain) +# mapper.connect("/services/{path_info:.*}", controller=ec2.API(), +# conditions=ec2api_subdomain) + mrh = metadatarequesthandler.MetadataRequestHandler() for s in ['/latest', '/2009-04-04', diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index c5cbe21ef117..4bdb14d04bc1 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -152,9 +152,10 @@ class Controller(wsgi.Controller): try: ctxt = req.environ['nova.context'] - self.compute_api.update_instance(ctxt, - id, - **update_dict) + # The ID passed in is actually the internal_id of the + # instance, not the value of the id column in the DB. + instance = self.compute_api.get_instance(ctxt, id) + self.compute_api.update_instance(ctxt, instance.id, **update_dict) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() diff --git a/nova/compute/api.py b/nova/compute/api.py index 76a571d61cfa..1d21a46688f0 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -257,7 +257,7 @@ class ComputeAPI(base.Base): def get_instance(self, context, instance_id): return self.db.instance_get_by_internal_id(context, instance_id) - def _cast_compute_message(method, context, instance_id): + def _cast_compute_message(self, method, context, instance_id): """Generic handler for RPC calls.""" instance = self.get_instance(context, instance_id) host = instance['host'] diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 78f2d470440c..eaf8c7dff3c7 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -130,12 +130,18 @@ class VMOps(object): """Refactored out the common code of many methods that receive either a vm name or a vm instance, and want a vm instance in return. """ + vm = None try: - instance_name = instance_or_vm.name - vm = VMHelper.lookup(self._session, instance_name) + if instance_or_vm.startswith("OpaqueRef:"): + # Got passed an opaque ref; return it + return instance_or_vm + else: + # Must be the instance name + instance_name = instance_or_vm except AttributeError: - # A vm opaque ref was passed - vm = instance_or_vm + # Not a string; must be a vm instance + instance_name = instance_or_vm.name + vm = VMHelper.lookup(self._session, instance_name) if vm is None: raise Exception(_('Instance not present %s') % instance_name) return vm @@ -201,6 +207,9 @@ class VMOps(object): instead of the more advanced one in M2Crypto for compatibility with the agent code. """ + + logging.error("ZZZZ RESET PASS CALLED") + # Need to uniquely identify this request. transaction_id = str(uuid.uuid4()) # The simple Diffie-Hellman class is used to manage key exchange. @@ -291,7 +300,7 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.resume', vm, False, True) self._wait_with_callback(task, callback) - def get_info(self, instance_id): + def get_info(self, instance): """Return data about VM instance""" vm = self._get_vm_opaque_ref(instance) rec = self._session.get_xenapi().VM.get_record(vm) From a3e12f5eb92921acc622ea7bd9097edeea0d40fd Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 6 Jan 2011 06:45:14 -0600 Subject: [PATCH 38/84] Renamed 'set_root_password' to 'set_admin_password' globally. --- nova/compute/api.py | 50 +++++++++++++------------------------- nova/compute/manager.py | 24 +++++++++--------- nova/tests/test_compute.py | 6 ++--- nova/virt/fake.py | 19 ++++++++------- nova/virt/xenapi/vmops.py | 20 +++++++-------- nova/virt/xenapi_conn.py | 6 ++--- 6 files changed, 56 insertions(+), 69 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 1d21a46688f0..0a7b802d6cef 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -190,7 +190,7 @@ class ComputeAPI(base.Base): """ try: db.security_group_get_by_name(context, context.project_id, - 'default') + 'default') except exception.NotFound: values = {'name': 'default', 'description': 'default', @@ -258,70 +258,54 @@ class ComputeAPI(base.Base): return self.db.instance_get_by_internal_id(context, instance_id) def _cast_compute_message(self, method, context, instance_id): - """Generic handler for RPC calls.""" + """Generic handler for RPC calls to compute.""" instance = self.get_instance(context, instance_id) host = instance['host'] rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": method, - "args": {"instance_id": instance['id']}}) + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {'method': method, 'args': {'instance_id': instance['id']}}) def snapshot(self, context, instance_id, name): """Snapshot the given instance.""" - self._cast_compute_message("snapshot_instance", context, instance_id) + self._cast_compute_message('snapshot_instance', context, instance_id) def reboot(self, context, instance_id): """Reboot the given instance.""" - self._cast_compute_message("reboot_instance", context, instance_id) + self._cast_compute_message('reboot_instance', context, instance_id) def pause(self, context, instance_id): """Pause the given instance.""" - self._cast_compute_message("pause_instance", context, instance_id) + self._cast_compute_message('pause_instance', context, instance_id) def unpause(self, context, instance_id): """Unpause the given instance.""" - self._cast_compute_message("unpause_instance", context, instance_id) + self._cast_compute_message('unpause_instance', context, instance_id) def get_diagnostics(self, context, instance_id): """Retrieve diagnostics for the given instance.""" - instance = self.db.instance_get_by_internal_id(context, instance_id) - host = instance["host"] - return rpc.call(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "get_diagnostics", - "args": {"instance_id": instance["id"]}}) + self._cast_compute_message('get_diagnostics', context, instance_id) def get_actions(self, context, instance_id): """Retrieve actions for the given instance.""" instance = self.db.instance_get_by_internal_id(context, instance_id) - return self.db.instance_get_actions(context, instance["id"]) + return self.db.instance_get_actions(context, instance['id']) def suspend(self, context, instance_id): """suspend the instance with instance_id""" - instance = self.db.instance_get_by_internal_id(context, instance_id) - host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "suspend_instance", - "args": {"instance_id": instance['id']}}) + self._cast_compute_message('suspend_instance', context, instance_id) def resume(self, context, instance_id): """resume the instance with instance_id""" - instance = self.db.instance_get_by_internal_id(context, instance_id) - host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "resume_instance", - "args": {"instance_id": instance['id']}}) + self._cast_compute_message('resume_instance', context, instance_id) def rescue(self, context, instance_id): """Rescue the given instance.""" - self._cast_compute_message("rescue_instance", context, instance_id) + self._cast_compute_message('rescue_instance', context, instance_id) def unrescue(self, context, instance_id): """Unrescue the given instance.""" - self._cast_compute_message("unrescue_instance", context, instance_id) + self._cast_compute_message('unrescue_instance', context, instance_id) - def reset_root_password(self, context, instance_id): - """Reset the root/admin pw for the given instance.""" - self._cast_compute_message("reset_root_password", context, instance_id) + def set_admin_password(self, context, instance_id): + """Set the root/admin password for the given instance.""" + self._cast_compute_message('set_admin_password', context, instance_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 6f4d145897af..b8bf91530645 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -53,6 +53,8 @@ flags.DEFINE_string('compute_driver', 'nova.virt.connection.get_connection', 'Driver to use for controlling virtualization') flags.DEFINE_string('stub_network', False, 'Stub network related code') +flags.DEFINE_integer('password_length', 12, + 'Length of generated admin passwords') class ComputeManager(manager.Manager): @@ -248,27 +250,27 @@ class ComputeManager(manager.Manager): self.driver.snapshot(instance_ref, name) @exception.wrap_exception - def reset_root_password(self, context, instance_id, new_pass=None): - """Reset the root/admin password for an instance on this server.""" + def set_admin_password(self, context, instance_id, new_pass=None): + """Set the root/admin password for an instance on this server.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) self._update_state(context, instance_id) if instance_ref['state'] != power_state.RUNNING: logging.warn('trying to reset the password on a non-running ' - 'instance: %s (state: %s excepted: %s)', - instance_ref['internal_id'], - instance_ref['state'], - power_state.RUNNING) + 'instance: %s (state: %s expected: %s)', + instance_ref['internal_id'], + instance_ref['state'], + power_state.RUNNING) - logging.debug('instance %s: resetting root password', + logging.debug('instance %s: setting admin password', instance_ref['name']) self.db.instance_set_state(context, instance_id, - power_state.NOSTATE, 'resetting_password') + power_state.NOSTATE, 'setting_password') if new_pass is None: - # Generate a random, 12-character password - new_pass = self._generate_password(12) - self.driver.reset_root_password(instance_ref, new_pass) + # Generate a random password + new_pass = self._generate_password(FLAGS.password_length) + self.driver.set_admin_password(instance_ref, new_pass) self._update_state(context, instance_id) def _generate_password(self, length=20): diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 7c258e636138..88e14d7dfa8e 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -151,11 +151,11 @@ class ComputeTestCase(test.TestCase): self.compute.reboot_instance(self.context, instance_id) self.compute.terminate_instance(self.context, instance_id) - def test_reset_root_password(self): - """Ensure instance can have its root password reset""" + def test_set_admin_password(self): + """Ensure instance can have its admin password set""" instance_id = self._create_instance() self.compute.run_instance(self.context, instance_id) - self.compute.reset_root_password(self.context, instance_id) + self.compute.set_admin_password(self.context, instance_id) self.compute.terminate_instance(self.context, instance_id) def test_snapshot(self): diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 2b9cf1ca3469..2d4b0a3d778a 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -98,7 +98,7 @@ class FakeConnection(object): the new instance. The work will be done asynchronously. This function returns a - Deferred that allows the caller to detect when it is complete. + task that allows the caller to detect when it is complete. Once this successfully completes, the instance should be running (power_state.RUNNING). @@ -122,7 +122,7 @@ class FakeConnection(object): The second parameter is the name of the snapshot. The work will be done asynchronously. This function returns a - Deferred that allows the caller to detect when it is complete. + task that allows the caller to detect when it is complete. """ pass @@ -134,19 +134,20 @@ class FakeConnection(object): and so the instance is being specified as instance.name. The work will be done asynchronously. This function returns a - Deferred that allows the caller to detect when it is complete. + task that allows the caller to detect when it is complete. """ pass - def reset_root_password(self, instance, new_pass): + def set_admin_password(self, instance, new_pass): """ - Reset the root password on the specified instance. + Set the root password on the specified instance. - The given parameter is an instance of nova.compute.service.Instance, - and so the instance is being specified as instance.name. + The first parameter is an instance of nova.compute.service.Instance, + and so the instance is being specified as instance.name. The second + parameter is the value of the new password. The work will be done asynchronously. This function returns a - Deferred that allows the caller to detect when it is complete. + task that allows the caller to detect when it is complete. """ pass @@ -194,7 +195,7 @@ class FakeConnection(object): and so the instance is being specified as instance.name. The work will be done asynchronously. This function returns a - Deferred that allows the caller to detect when it is complete. + task that allows the caller to detect when it is complete. """ del self.instances[instance.name] diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index eaf8c7dff3c7..64855001b811 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -198,17 +198,16 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.clean_reboot', vm) self._session.wait_for_task(instance.id, task) - def reset_root_password(self, instance, new_pass): - """Reset the root/admin password on the VM instance. This is - done via an agent running on the VM. Communication between - nova and the agent is done via writing xenstore records. Since - communication is done over the XenAPI RPC calls, we need to - encrypt the password. We're using a simple Diffie-Hellman class - instead of the more advanced one in M2Crypto for compatibility - with the agent code. + def set_admin_password(self, instance, new_pass): + """Set the root/admin password on the VM instance. This is done via + an agent running on the VM. Communication between nova and the agent + is done via writing xenstore records. Since communication is done over + the XenAPI RPC calls, we need to encrypt the password. We're using a + simple Diffie-Hellman class instead of the more advanced one in + M2Crypto for compatibility with the agent code. """ - logging.error("ZZZZ RESET PASS CALLED") + logging.error("ZZZZ SET PASS CALLED") # Need to uniquely identify this request. transaction_id = str(uuid.uuid4()) @@ -219,7 +218,8 @@ class VMOps(object): resp_dict = json.loads(resp) # Successful return code from key_init is 'D0' if resp_dict['returncode'] != 'D0': - # There was some sort of error + # There was some sort of error; the message will contain + # a description of the error. raise RuntimeError(resp_dict['message']) agent_pub = int(resp_dict['message']) dh.compute_shared(agent_pub) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index f4dd7055c981..0cba2812de05 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -144,9 +144,9 @@ class XenAPIConnection(object): """Reboot VM instance""" self._vmops.reboot(instance) - def reset_root_password(self, instance, new_pass): - """Reset the root/admin password on the VM instance""" - self._vmops.reset_root_password(instance, new_pass) + def set_admin_password(self, instance, new_pass): + """Set the root/admin password on the VM instance""" + self._vmops.set_admin_password(instance, new_pass) def destroy(self, instance): """Destroy VM instance""" From 0209ad587b2d8d35a7abdf60ca9b33391cab4a83 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 6 Jan 2011 07:21:11 -0600 Subject: [PATCH 39/84] merged trunk changes --- nova/compute/api.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 476667fa8de9..19cdf2d0a618 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -61,13 +61,11 @@ class API(base.Base): def get_network_topic(self, context, instance_id): try: -<<<<<<< TREE instance = self.get_instance(context, instance_id) + # TODO (dabo) Need to verify whether an internal_id or a db id + # id being passed; use get_instance or get, respectively. + #instance = self.get(context, instance_id) except exception.NotFound, e: -======= - instance = self.get(context, instance_id) - except exception.NotFound as e: ->>>>>>> MERGE-SOURCE logging.warning("Instance %d was not found in get_network_topic", instance_id) raise e From 3d70b49a1c17bccfc6163198b2d99efb9a9829a7 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 6 Jan 2011 13:02:32 -0600 Subject: [PATCH 40/84] commit before merging trunk --- nova/api/ec2/cloud.py | 2 +- nova/api/openstack/servers.py | 14 +++++++++++--- nova/compute/api.py | 18 ++++-------------- nova/compute/manager.py | 5 +++-- nova/db/api.py | 5 +++++ nova/db/sqlalchemy/api.py | 23 +++++++++++++++++++++++ nova/virt/xenapi/vmops.py | 7 +++++++ nova/virt/xenapi_conn.py | 3 +++ 8 files changed, 57 insertions(+), 20 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 0c0027287228..6619b545292c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -592,7 +592,7 @@ class CloudController(object): return {'reservationSet': self._format_instances(context)} def _format_run_instances(self, context, reservation_id): - i = self._format_instances(context, reservation_id) + i = self._format_instances(context, reservation_id=reservation_id) assert len(i) == 1 return i[0] diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 05ba56c7969c..e0513e4c10ed 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -104,7 +104,7 @@ class Controller(wsgi.Controller): def show(self, req, id): """ Returns server details by server id """ try: - instance = self.compute_api.get(req.environ['nova.context'], id) + instance = self.compute_api.get_instance(req.environ['nova.context'], id) return _translate_detail_keys(instance) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) @@ -142,8 +142,10 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) update_dict = {} + func = None if 'adminPass' in inst_dict['server']: update_dict['admin_pass'] = inst_dict['server']['adminPass'] + func = self.compute_api.set_admin_password if 'name' in inst_dict['server']: update_dict['display_name'] = inst_dict['server']['name'] @@ -152,9 +154,15 @@ class Controller(wsgi.Controller): # The ID passed in is actually the internal_id of the # instance, not the value of the id column in the DB. instance = self.compute_api.get_instance(ctxt, id) - self.compute_api.update_instance(ctxt, instance.id, **update_dict) + self.compute_api.update(ctxt, instance.id, **update_dict) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) + + logging.error("ZZZZ func=%s" % func) + logging.error("ZZZZ UPD=%s" % id) + + if func: + func(ctxt, id) return exc.HTTPNoContent() def action(self, req, id): @@ -225,4 +233,4 @@ class Controller(wsgi.Controller): def actions(self, req, id): """Permit Admins to retrieve server actions.""" ctxt = req.environ["nova.context"] - return self.compute_api.get_actions(ctxt, id) + return self.compute_api.get_actions(ctxt, id_val) diff --git a/nova/compute/api.py b/nova/compute/api.py index 19cdf2d0a618..a4345f337fb2 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -62,9 +62,6 @@ class API(base.Base): def get_network_topic(self, context, instance_id): try: instance = self.get_instance(context, instance_id) - # TODO (dabo) Need to verify whether an internal_id or a db id - # id being passed; use get_instance or get, respectively. - #instance = self.get(context, instance_id) except exception.NotFound, e: logging.warning("Instance %d was not found in get_network_topic", instance_id) @@ -223,10 +220,6 @@ class API(base.Base): logging.debug('Going to try and terminate %s' % instance_id) try: instance = self.get_instance(context, instance_id) - #TODO: (dabo) resolve that this is the correct call: - # get_instance vs. get. Depends on whether we get an internal_id - # or an actual db id. - #instance = self.get(context, instance_id) except exception.NotFound, e: logging.warning(_('Instance % was not found during terminate'), instance_id) @@ -252,7 +245,7 @@ class API(base.Base): else: self.db.instance_destroy(context, instance_id) - def get(self, context, instance_id): + def get_instance(self, context, instance_id): """Get a single instance with the given ID.""" return self.db.instance_get_by_id(context, instance_id) @@ -276,9 +269,6 @@ class API(base.Base): project_id) return self.db.instance_get_all(context) - def get_instance(self, context, instance_id): - return self.db.instance_get_by_internal_id(context, instance_id) - def _cast_compute_message(self, method, context, instance_id): """Generic handler for RPC calls to compute.""" instance = self.get_instance(context, instance_id) @@ -309,7 +299,7 @@ class API(base.Base): def get_actions(self, context, instance_id): """Retrieve actions for the given instance.""" - instance = self.db.instance_get_by_internal_id(context, instance_id) + instance = self.db.instance_get_by_id(context, instance_id) return self.db.instance_get_actions(context, instance['id']) def suspend(self, context, instance_id): @@ -337,7 +327,7 @@ class API(base.Base): raise exception.ApiError(_("Invalid device specified: %s. " "Example device: /dev/vdb") % device) self.volume_api.check_attach(context, volume_id) - instance = self.get(context, instance_id) + instance = self.get_instance(context, instance_id) host = instance['host'] rpc.cast(context, self.db.queue_get_for(context, FLAGS.compute_topic, host), @@ -360,6 +350,6 @@ class API(base.Base): return instance def associate_floating_ip(self, context, instance_id, address): - instance = self.get(context, instance_id) + instance = self.get_instance(context, instance_id) self.network_api.associate_floating_ip(context, address, instance['fixed_ip']) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3e6d3eab0f5c..201fffc687b6 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -83,7 +83,8 @@ class ComputeManager(manager.Manager): # FIXME(ja): include other fields from state? instance_ref = self.db.instance_get(context, instance_id) try: - info = self.driver.get_info(instance_ref['name']) + #info = self.driver.get_info(instance_ref['name']) + info = self.driver.get_info(instance_ref) state = info['state'] except exception.NotFound: state = power_state.NOSTATE @@ -259,7 +260,7 @@ class ComputeManager(manager.Manager): if instance_ref['state'] != power_state.RUNNING: logging.warn('trying to reset the password on a non-running ' 'instance: %s (state: %s expected: %s)', - instance_ref['internal_id'], + instance_ref['id'], instance_ref['state'], power_state.RUNNING) diff --git a/nova/db/api.py b/nova/db/api.py index 0fa5eb1e8e42..f5c7eab91fe8 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -358,6 +358,11 @@ def instance_get_by_id(context, instance_id): return IMPL.instance_get_by_id(context, instance_id) +def instance_get_by_internal_id(context, internal_id): + """Get an instance by internal id.""" + return IMPL.instance_get_by_internal_id(context, internal_id) + + def instance_is_vpn(context, instance_id): """True if instance is a vpn.""" return IMPL.instance_is_vpn(context, instance_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index aaa07e3c9291..34c73490ed74 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -670,6 +670,29 @@ def instance_get(context, instance_id, session=None): return result +@require_context +def instance_get_by_internal_id(context, internal_id): + session = get_session() + + if is_admin_context(context): + result = session.query(models.Instance).\ + options(joinedload('security_groups')).\ + filter_by(internal_id=internal_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() + elif is_user_context(context): + result = session.query(models.Instance).\ + options(joinedload('security_groups')).\ + filter_by(project_id=context.project_id).\ + filter_by(internal_id=internal_id).\ + filter_by(deleted=False).\ + first() + if not result: + raise exception.NotFound(_('Instance %s not found') % (internal_id)) + + return result + + @require_admin_context def instance_get_all(context): session = get_session() diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 64855001b811..e092601b9fcb 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -130,17 +130,21 @@ class VMOps(object): """Refactored out the common code of many methods that receive either a vm name or a vm instance, and want a vm instance in return. """ + logging.error("ZZZZ opaq instance_or_vm=%s" % instance_or_vm) vm = None try: if instance_or_vm.startswith("OpaqueRef:"): + logging.error("ZZZZ opaq startswith") # Got passed an opaque ref; return it return instance_or_vm else: # Must be the instance name + logging.error("ZZZZ opaq inst name") instance_name = instance_or_vm except AttributeError: # Not a string; must be a vm instance instance_name = instance_or_vm.name + logging.error("ZZZZ opaq instance, name=%s" % instance_name) vm = VMHelper.lookup(self._session, instance_name) if vm is None: raise Exception(_('Instance not present %s') % instance_name) @@ -302,6 +306,9 @@ class VMOps(object): def get_info(self, instance): """Return data about VM instance""" + + logging.error("ZZZZ get_info instance=%s" % instance) + vm = self._get_vm_opaque_ref(instance) rec = self._session.get_xenapi().VM.get_record(vm) return VMHelper.compile_info(rec) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 0cba2812de05..0ceac8c97abe 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -170,6 +170,9 @@ class XenAPIConnection(object): def get_info(self, instance_id): """Return data about VM instance""" + + logging.error("ZZZZ conn get_info id=%s" % instance_id) + return self._vmops.get_info(instance_id) def get_diagnostics(self, instance): From e66f3017373dcf9135c53ae4d510b0b2a5dcecf0 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 6 Jan 2011 15:53:11 -0600 Subject: [PATCH 41/84] Got the basic 'set admin password' stuff working --- nova/api/openstack/servers.py | 17 +++++++------- nova/compute/manager.py | 3 --- nova/exception.py | 4 ++++ nova/virt/xenapi/vmops.py | 22 ++++++++----------- nova/virt/xenapi_conn.py | 16 ++------------ .../etc/xapi.d/plugins/{agent.py => agent} | 22 +++++++++++++++---- 6 files changed, 41 insertions(+), 43 deletions(-) rename plugins/xenserver/xenapi/etc/xapi.d/plugins/{agent.py => agent} (93%) mode change 100644 => 100755 diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index e0513e4c10ed..bc89f696cb49 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -104,7 +104,8 @@ class Controller(wsgi.Controller): def show(self, req, id): """ Returns server details by server id """ try: - instance = self.compute_api.get_instance(req.environ['nova.context'], id) + instance = self.compute_api.get_instance( + req.environ['nova.context'], id) return _translate_detail_keys(instance) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) @@ -141,6 +142,7 @@ class Controller(wsgi.Controller): if not inst_dict: return faults.Fault(exc.HTTPUnprocessableEntity()) + ctxt = req.environ['nova.context'] update_dict = {} func = None if 'adminPass' in inst_dict['server']: @@ -148,21 +150,18 @@ class Controller(wsgi.Controller): func = self.compute_api.set_admin_password if 'name' in inst_dict['server']: update_dict['display_name'] = inst_dict['server']['name'] - + if func: + try: + func(ctxt, id) + except exception.TimeoutException, e: + return exc.HTTPRequestTimeout() try: - ctxt = req.environ['nova.context'] # The ID passed in is actually the internal_id of the # instance, not the value of the id column in the DB. instance = self.compute_api.get_instance(ctxt, id) self.compute_api.update(ctxt, instance.id, **update_dict) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) - - logging.error("ZZZZ func=%s" % func) - logging.error("ZZZZ UPD=%s" % id) - - if func: - func(ctxt, id) return exc.HTTPNoContent() def action(self, req, id): diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 201fffc687b6..52acfebea21f 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -83,7 +83,6 @@ class ComputeManager(manager.Manager): # FIXME(ja): include other fields from state? instance_ref = self.db.instance_get(context, instance_id) try: - #info = self.driver.get_info(instance_ref['name']) info = self.driver.get_info(instance_ref) state = info['state'] except exception.NotFound: @@ -266,8 +265,6 @@ class ComputeManager(manager.Manager): logging.debug('instance %s: setting admin password', instance_ref['name']) - self.db.instance_set_state(context, instance_id, - power_state.NOSTATE, 'setting_password') if new_pass is None: # Generate a random password new_pass = self._generate_password(FLAGS.password_length) diff --git a/nova/exception.py b/nova/exception.py index 277033e0f324..52bf2a2a7eb0 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -77,6 +77,10 @@ class InvalidInputException(Error): pass +class TimeoutException(Error): + pass + + def wrap_exception(f): def _wrap(*args, **kw): try: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index e092601b9fcb..5f1654a49008 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -130,21 +130,17 @@ class VMOps(object): """Refactored out the common code of many methods that receive either a vm name or a vm instance, and want a vm instance in return. """ - logging.error("ZZZZ opaq instance_or_vm=%s" % instance_or_vm) vm = None try: if instance_or_vm.startswith("OpaqueRef:"): - logging.error("ZZZZ opaq startswith") # Got passed an opaque ref; return it return instance_or_vm else: # Must be the instance name - logging.error("ZZZZ opaq inst name") instance_name = instance_or_vm except AttributeError: # Not a string; must be a vm instance instance_name = instance_or_vm.name - logging.error("ZZZZ opaq instance, name=%s" % instance_name) vm = VMHelper.lookup(self._session, instance_name) if vm is None: raise Exception(_('Instance not present %s') % instance_name) @@ -210,9 +206,6 @@ class VMOps(object): simple Diffie-Hellman class instead of the more advanced one in M2Crypto for compatibility with the agent code. """ - - logging.error("ZZZZ SET PASS CALLED") - # Need to uniquely identify this request. transaction_id = str(uuid.uuid4()) # The simple Diffie-Hellman class is used to manage key exchange. @@ -306,9 +299,6 @@ class VMOps(object): def get_info(self, instance): """Return data about VM instance""" - - logging.error("ZZZZ get_info instance=%s" % instance) - vm = self._get_vm_opaque_ref(instance) rec = self._session.get_xenapi().VM.get_record(vm) return VMHelper.compile_info(rec) @@ -370,13 +360,14 @@ class VMOps(object): def _make_agent_call(self, method, vm, path, addl_args={}): """Abstracts out the interaction with the agent xenapi plugin.""" - return self._make_plugin_call('agent.py', method=method, vm=vm, + return self._make_plugin_call('agent', method=method, vm=vm, path=path, addl_args=addl_args) def _make_plugin_call(self, plugin, method, vm, path, addl_args={}): """Abstracts out the process of calling a method of a xenapi plugin. Any errors raised by the plugin will in turn raise a RuntimeError here. """ + instance_id = vm.id vm = self._get_vm_opaque_ref(vm) rec = self._session.get_xenapi().VM.get_record(vm) args = {'dom_id': rec['domid'], 'path': path} @@ -386,9 +377,14 @@ class VMOps(object): args['testing_mode'] = 'true' try: task = self._session.async_call_plugin(plugin, method, args) - ret = self._session.wait_for_task(0, task) + ret = self._session.wait_for_task(instance_id, task) except self.XenAPI.Failure, e: - raise RuntimeError("%s" % e.details[-1]) + err_trace = e.details[-1] + err_msg = err_trace.splitlines()[-1] + if 'TIMEOUT:' in err_msg: + raise exception.TimeoutException(err_msg) + else: + raise RuntimeError("%s" % e.details[-1]) return ret def add_to_xenstore(self, vm, path, key, value): diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 0ceac8c97abe..c9428e3a6877 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -170,9 +170,6 @@ class XenAPIConnection(object): def get_info(self, instance_id): """Return data about VM instance""" - - logging.error("ZZZZ conn get_info id=%s" % instance_id) - return self._vmops.get_info(instance_id) def get_diagnostics(self, instance): @@ -251,7 +248,8 @@ class XenAPISession(object): def _poll_task(self, id, task, done): """Poll the given XenAPI task, and fire the given action if we - get a result.""" + get a result. + """ try: name = self._session.xenapi.task.get_name_label(task) status = self._session.xenapi.task.get_status(task) @@ -278,16 +276,6 @@ class XenAPISession(object): error_info)) done.send_exception(self.XenAPI.Failure(error_info)) db.instance_action_create(context.get_admin_context(), action) -# import sqlalchemy -# from sqlalchemy.exc import IntegrityError as IntegrityError -# try: -# db.instance_action_create(context.get_admin_context(), action) -# except IntegrityError: -# # Some methods don't pass unique IDs, so the call to -# # instance_action_create() will raise IntegrityErrors. Rather -# # than bomb out, I'm explicitly silencing them so that the -# # code can continue to work until they fix that method. -# pass except self.XenAPI.Failure, exc: logging.warn(exc) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent.py b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent old mode 100644 new mode 100755 similarity index 93% rename from plugins/xenserver/xenapi/etc/xapi.d/plugins/agent.py rename to plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index 4b072ce67397..244509f3fc3f --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent.py +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -42,9 +42,16 @@ AGENT_TIMEOUT = 30 PRETEND_SECRET = 11111 +def jsonify(fnc): + def wrapper(*args, **kwargs): + return json.dumps(fnc(*args, **kwargs)) + return wrapper + + class TimeoutError(StandardError): pass + class SimpleDH(object): """This class wraps all the functionality needed to implement basic Diffie-Hellman-Merkle key exchange in Python. It features @@ -138,6 +145,8 @@ def _run_command(cmd): raise PluginError(err) return proc.stdout.read() + +@jsonify def key_init(self, arg_dict): """Handles the Diffie-Hellman key exchange with the agent to establish the shared secret key used to encrypt/decrypt sensitive @@ -152,7 +161,7 @@ def key_init(self, arg_dict): pretend = SimpleDH(secret=PRETEND_SECRET) shared = pretend.compute_shared(pub) # Simulate the agent's response - ret = '{ "returncode": "D0", "message": "%s", "shared": "%s" }' % (pretend.get_public(), shared) + ret = {"returncode": "D0", "message": "%s", "shared": "%s"} % (pretend.get_public(), shared) return ret arg_dict["path"] = "data/host/%s" % request_id xenstore.write_record(self, arg_dict) @@ -162,6 +171,8 @@ def key_init(self, arg_dict): raise PluginError("%s" % e) return resp + +@jsonify def password(self, arg_dict): """Writes a request to xenstore that tells the agent to set the root password for the given VM. The password should be @@ -176,7 +187,7 @@ def password(self, arg_dict): pretend = SimpleDH(secret=PRETEND_SECRET) pretend.compute_shared(pub) pw = pretend.decrypt(enc_pass) - ret = '{ "returncode": "0", "message": "%s" }' % pw + ret = {"returncode": "0", "message": "%s"} % pw return ret arg_dict["value"] = json.dumps({"name": "password", "value": enc_pass}) request_id = arg_dict["id"] @@ -188,6 +199,7 @@ def password(self, arg_dict): raise PluginError("%s" % e) return resp + def _wait_for_agent(self, request_id, arg_dict): """Periodically checks xenstore for a response from the agent. The request is always written to 'data/host/{id}', and @@ -205,10 +217,12 @@ def _wait_for_agent(self, request_id, arg_dict): # First, delete the request record arg_dict["path"] = "data/host/%s" % request_id xenstore.delete_record(self, arg_dict) - raise TimeoutError("No response from agent within %s seconds." % + raise TimeoutError("TIMEOUT: No response from agent within %s seconds." % AGENT_TIMEOUT) ret = xenstore.read_record(self, arg_dict) - if ret != "None": + # Note: the response for None with be a string that includes + # double quotes. + if ret != '"None"': # The agent responded return ret else: From dea9f51d65ce0d5c3c4ea328a2231499c71719d6 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 6 Jan 2011 17:37:02 -0500 Subject: [PATCH 42/84] various cleanup and fixes --- nova/api/openstack/servers.py | 7 ++----- nova/compute/api.py | 2 +- nova/db/sqlalchemy/api.py | 23 ----------------------- 3 files changed, 3 insertions(+), 29 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index bc89f696cb49..9eaef2b40ca3 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -156,10 +156,7 @@ class Controller(wsgi.Controller): except exception.TimeoutException, e: return exc.HTTPRequestTimeout() try: - # The ID passed in is actually the internal_id of the - # instance, not the value of the id column in the DB. - instance = self.compute_api.get_instance(ctxt, id) - self.compute_api.update(ctxt, instance.id, **update_dict) + self.compute_api.update(ctxt, id, **update_dict) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() @@ -232,4 +229,4 @@ class Controller(wsgi.Controller): def actions(self, req, id): """Permit Admins to retrieve server actions.""" ctxt = req.environ["nova.context"] - return self.compute_api.get_actions(ctxt, id_val) + return self.compute_api.get_actions(ctxt, id) diff --git a/nova/compute/api.py b/nova/compute/api.py index a4345f337fb2..232eddb641ba 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -217,7 +217,7 @@ class API(base.Base): return self.db.instance_update(context, instance_id, kwargs) def delete(self, context, instance_id): - logging.debug('Going to try and terminate %s' % instance_id) + logging.debug('Going to try to terminate %s' % instance_id) try: instance = self.get_instance(context, instance_id) except exception.NotFound, e: diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 34c73490ed74..aaa07e3c9291 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -670,29 +670,6 @@ def instance_get(context, instance_id, session=None): return result -@require_context -def instance_get_by_internal_id(context, internal_id): - session = get_session() - - if is_admin_context(context): - result = session.query(models.Instance).\ - options(joinedload('security_groups')).\ - filter_by(internal_id=internal_id).\ - filter_by(deleted=can_read_deleted(context)).\ - first() - elif is_user_context(context): - result = session.query(models.Instance).\ - options(joinedload('security_groups')).\ - filter_by(project_id=context.project_id).\ - filter_by(internal_id=internal_id).\ - filter_by(deleted=False).\ - first() - if not result: - raise exception.NotFound(_('Instance %s not found') % (internal_id)) - - return result - - @require_admin_context def instance_get_all(context): session = get_session() From e01123943e7fbe81d7cb40325cde6c517bb2ffd9 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 7 Jan 2011 08:36:11 -0500 Subject: [PATCH 43/84] grabbed the get_info fix from my other branch --- nova/virt/xenapi/vmops.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 5f1654a49008..65d0360c88d2 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -138,8 +138,9 @@ class VMOps(object): else: # Must be the instance name instance_name = instance_or_vm - except AttributeError: + except (AttributeError, KeyError): # Not a string; must be a vm instance + # Note the the KeyError will only happen with fakes.py instance_name = instance_or_vm.name vm = VMHelper.lookup(self._session, instance_name) if vm is None: @@ -297,9 +298,12 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.resume', vm, False, True) self._wait_with_callback(task, callback) - def get_info(self, instance): + def get_info(self, instance_id): """Return data about VM instance""" - vm = self._get_vm_opaque_ref(instance) + vm = VMHelper.lookup(self._session, instance_id) + if vm is None: + raise exception.NotFound(_('Instance not' + ' found %s') % instance_id) rec = self._session.get_xenapi().VM.get_record(vm) return VMHelper.compile_info(rec) From eaa5b5994891eee0280b750dff221a4b54932eb9 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 7 Jan 2011 10:23:48 -0600 Subject: [PATCH 44/84] getting ready to push for merge prop --- nova/api/openstack/servers.py | 4 ++-- nova/compute/api.py | 18 +++++++++--------- nova/compute/manager.py | 3 +-- nova/virt/xenapi/vmops.py | 16 ++++++++++------ .../xenserver/xenapi/etc/xapi.d/plugins/agent | 4 ++-- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index a426a721dadf..280e2349efed 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -104,7 +104,7 @@ class Controller(wsgi.Controller): def show(self, req, id): """ Returns server details by server id """ try: - instance = self.compute_api.get_instance( + instance = self.compute_api.get( req.environ['nova.context'], id) return _translate_detail_keys(instance) except exception.NotFound: @@ -158,7 +158,7 @@ class Controller(wsgi.Controller): try: # The ID passed in is actually the internal_id of the # instance, not the value of the id column in the DB. - instance = self.compute_api.get_instance(ctxt, id) + instance = self.compute_api.get(ctxt, id) self.compute_api.update(ctxt, instance.id, **update_dict) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) diff --git a/nova/compute/api.py b/nova/compute/api.py index 78ffcca7abc2..106c3f7f0227 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -61,7 +61,7 @@ class API(base.Base): def get_network_topic(self, context, instance_id): try: - instance = self.get_instance(context, instance_id) + instance = self.get(context, instance_id) except exception.NotFound, e: logging.warning("Instance %d was not found in get_network_topic", instance_id) @@ -220,7 +220,7 @@ class API(base.Base): def delete(self, context, instance_id): logging.debug('Going to try and terminate %s' % instance_id) try: - instance = self.get_instance(context, instance_id) + instance = self.get(context, instance_id) except exception.NotFound, e: logging.warning(_('Instance % was not found during terminate'), instance_id) @@ -246,7 +246,7 @@ class API(base.Base): else: self.db.instance_destroy(context, instance_id) - def get_instance(self, context, instance_id): + def get(self, context, instance_id): """Get a single instance with the given ID.""" return self.db.instance_get_by_id(context, instance_id) @@ -272,7 +272,7 @@ class API(base.Base): def _cast_compute_message(self, method, context, instance_id): """Generic handler for RPC calls to compute.""" - instance = self.get_instance(context, instance_id) + instance = self.get(context, instance_id) host = instance['host'] rpc.cast(context, self.db.queue_get_for(context, FLAGS.compute_topic, host), @@ -328,7 +328,7 @@ class API(base.Base): lock the instance with instance_id """ - instance = self.get_instance(context, instance_id) + instance = self.get(context, instance_id) host = instance['host'] rpc.cast(context, self.db.queue_get_for(context, FLAGS.compute_topic, host), @@ -340,7 +340,7 @@ class API(base.Base): unlock the instance with instance_id """ - instance = self.get_instance(context, instance_id) + instance = self.get(context, instance_id) host = instance['host'] rpc.cast(context, self.db.queue_get_for(context, FLAGS.compute_topic, host), @@ -352,7 +352,7 @@ class API(base.Base): return the boolean state of (instance with instance_id)'s lock """ - instance = self.get_instance(context, instance_id) + instance = self.get(context, instance_id) return instance['locked'] def attach_volume(self, context, instance_id, volume_id, device): @@ -360,7 +360,7 @@ class API(base.Base): raise exception.ApiError(_("Invalid device specified: %s. " "Example device: /dev/vdb") % device) self.volume_api.check_attach(context, volume_id) - instance = self.get_instance(context, instance_id) + instance = self.get(context, instance_id) host = instance['host'] rpc.cast(context, self.db.queue_get_for(context, FLAGS.compute_topic, host), @@ -383,6 +383,6 @@ class API(base.Base): return instance def associate_floating_ip(self, context, instance_id, address): - instance = self.get_instance(context, instance_id) + instance = self.get(context, instance_id) self.network_api.associate_floating_ip(context, address, instance['fixed_ip']) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 10219833befa..5d677b023f27 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -289,8 +289,6 @@ class ComputeManager(manager.Manager): """Set the root/admin password for an instance on this server.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - self._update_state(context, instance_id) - if instance_ref['state'] != power_state.RUNNING: logging.warn('trying to reset the password on a non-running ' 'instance: %s (state: %s expected: %s)', @@ -303,6 +301,7 @@ class ComputeManager(manager.Manager): if new_pass is None: # Generate a random password new_pass = self._generate_password(FLAGS.password_length) + self.driver.set_admin_password(instance_ref, new_pass) self._update_state(context, instance_id) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 5f1654a49008..3df561e7c1b4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -137,13 +137,17 @@ class VMOps(object): return instance_or_vm else: # Must be the instance name - instance_name = instance_or_vm - except AttributeError: - # Not a string; must be a vm instance - instance_name = instance_or_vm.name - vm = VMHelper.lookup(self._session, instance_name) + instance = instance_or_vm + except (AttributeError, KeyError): + # Note the the KeyError will only happen with fakes.py + # Not a string; must be an ID or a vm instance + if isinstance(instance_or_vm, (int, long)): + instance = instance_or_vm + else: + instance = instance_or_vm.name + vm = VMHelper.lookup(self._session, instance) if vm is None: - raise Exception(_('Instance not present %s') % instance_name) + raise Exception(_('Instance not present %s') % instance) return vm def snapshot(self, instance, name): diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index 244509f3fc3f..ab5b98d1cd20 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -161,7 +161,7 @@ def key_init(self, arg_dict): pretend = SimpleDH(secret=PRETEND_SECRET) shared = pretend.compute_shared(pub) # Simulate the agent's response - ret = {"returncode": "D0", "message": "%s", "shared": "%s"} % (pretend.get_public(), shared) + ret = {"returncode": "D0", "message": pretend.get_public(), "shared": shared} return ret arg_dict["path"] = "data/host/%s" % request_id xenstore.write_record(self, arg_dict) @@ -187,7 +187,7 @@ def password(self, arg_dict): pretend = SimpleDH(secret=PRETEND_SECRET) pretend.compute_shared(pub) pw = pretend.decrypt(enc_pass) - ret = {"returncode": "0", "message": "%s"} % pw + ret = {"returncode": "0", "message": pw} return ret arg_dict["value"] = json.dumps({"name": "password", "value": enc_pass}) request_id = arg_dict["id"] From 147693e45c7be174c54e39160869ca9a83bb4fff Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 7 Jan 2011 11:04:53 -0600 Subject: [PATCH 45/84] Additional cleanup prior to pushing --- nova/api/__init__.py | 14 +---- nova/api/openstack/servers.py | 2 - nova/compute/api.py | 54 ++++++------------- nova/db/api.py | 5 -- nova/db/sqlalchemy/api.py | 23 -------- nova/virt/xenapi/vmops.py | 6 +++ nova/virt/xenapi_conn.py | 1 - .../xenserver/xenapi/etc/xapi.d/plugins/agent | 8 +-- 8 files changed, 29 insertions(+), 84 deletions(-) diff --git a/nova/api/__init__.py b/nova/api/__init__.py index ff936bed247f..26fed847bb15 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -59,23 +59,13 @@ class API(wsgi.Router): mapper.connect("/", controller=self.osapi_versions, conditions=osapi_subdomain) - mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API()) + mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API(), + conditions=osapi_subdomain) mapper.connect("/", controller=self.ec2api_versions, conditions=ec2api_subdomain) mapper.connect("/services/{path_info:.*}", controller=ec2.API(), conditions=ec2api_subdomain) - -# mapper.connect("/", controller=self.osapi_versions, -# conditions=osapi_subdomain) -# mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API(), -# conditions=osapi_subdomain) -# -# mapper.connect("/", controller=self.ec2api_versions, -# conditions=ec2api_subdomain) -# mapper.connect("/services/{path_info:.*}", controller=ec2.API(), -# conditions=ec2api_subdomain) - mrh = metadatarequesthandler.MetadataRequestHandler() for s in ['/latest', '/2009-04-04', diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 280e2349efed..6a35567ff832 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -156,8 +156,6 @@ class Controller(wsgi.Controller): except exception.TimeoutException, e: return exc.HTTPRequestTimeout() try: - # The ID passed in is actually the internal_id of the - # instance, not the value of the id column in the DB. instance = self.compute_api.get(ctxt, id) self.compute_api.update(ctxt, instance.id, **update_dict) except exception.NotFound: diff --git a/nova/compute/api.py b/nova/compute/api.py index 106c3f7f0227..a894a0ce388f 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -62,7 +62,7 @@ class API(base.Base): def get_network_topic(self, context, instance_id): try: instance = self.get(context, instance_id) - except exception.NotFound, e: + except exception.NotFound as e: logging.warning("Instance %d was not found in get_network_topic", instance_id) raise e @@ -195,7 +195,7 @@ class API(base.Base): """ try: db.security_group_get_by_name(context, context.project_id, - 'default') + 'default') except exception.NotFound: values = {'name': 'default', 'description': 'default', @@ -218,12 +218,12 @@ class API(base.Base): return self.db.instance_update(context, instance_id, kwargs) def delete(self, context, instance_id): - logging.debug('Going to try and terminate %s' % instance_id) + logging.debug('Going to try to terminate %s' % instance_id) try: instance = self.get(context, instance_id) - except exception.NotFound, e: + except exception.NotFound as e: logging.warning(_('Instance % was not found during terminate'), - instance_id) + instance_id) raise e if (instance['state_description'] == 'terminating'): @@ -239,10 +239,8 @@ class API(base.Base): host = instance['host'] if host: - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "terminate_instance", - "args": {"instance_id": instance_id}}) + self._cast_compute_message('terminate_instance', context, + instance_id) else: self.db.instance_destroy(context, instance_id) @@ -274,9 +272,9 @@ class API(base.Base): """Generic handler for RPC calls to compute.""" instance = self.get(context, instance_id) host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {'method': method, 'args': {'instance_id': instance['id']}}) + queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) + kwargs = {'method': method, 'args': {'instance_id': instance['id']}} + rpc.cast(context, queue, kwargs) def snapshot(self, context, instance_id, name): """Snapshot the given instance.""" @@ -300,8 +298,7 @@ class API(base.Base): def get_actions(self, context, instance_id): """Retrieve actions for the given instance.""" - instance = self.db.instance_get_by_id(context, instance_id) - return self.db.instance_get_actions(context, instance['id']) + return self.db.instance_get_actions(context, instance_id) def suspend(self, context, instance_id): """suspend the instance with instance_id""" @@ -324,34 +321,15 @@ class API(base.Base): self._cast_compute_message('set_admin_password', context, instance_id) def lock(self, context, instance_id): - """ - lock the instance with instance_id - - """ - instance = self.get(context, instance_id) - host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "lock_instance", - "args": {"instance_id": instance['id']}}) + """lock the instance with instance_id""" + self._cast_compute_message('lock_instance', context, instance_id) def unlock(self, context, instance_id): - """ - unlock the instance with instance_id - - """ - instance = self.get(context, instance_id) - host = instance['host'] - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "unlock_instance", - "args": {"instance_id": instance['id']}}) + """unlock the instance with instance_id""" + self._cast_compute_message('unlock_instance', context, instance_id) def get_lock(self, context, instance_id): - """ - return the boolean state of (instance with instance_id)'s lock - - """ + """return the boolean state of (instance with instance_id)'s lock""" instance = self.get(context, instance_id) return instance['locked'] diff --git a/nova/db/api.py b/nova/db/api.py index f5c7eab91fe8..0fa5eb1e8e42 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -358,11 +358,6 @@ def instance_get_by_id(context, instance_id): return IMPL.instance_get_by_id(context, instance_id) -def instance_get_by_internal_id(context, internal_id): - """Get an instance by internal id.""" - return IMPL.instance_get_by_internal_id(context, internal_id) - - def instance_is_vpn(context, instance_id): """True if instance is a vpn.""" return IMPL.instance_is_vpn(context, instance_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 0e5c14275bd5..45427597a268 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -670,29 +670,6 @@ def instance_get(context, instance_id, session=None): return result -@require_context -def instance_get_by_internal_id(context, internal_id): - session = get_session() - - if is_admin_context(context): - result = session.query(models.Instance).\ - options(joinedload('security_groups')).\ - filter_by(internal_id=internal_id).\ - filter_by(deleted=can_read_deleted(context)).\ - first() - elif is_user_context(context): - result = session.query(models.Instance).\ - options(joinedload('security_groups')).\ - filter_by(project_id=context.project_id).\ - filter_by(internal_id=internal_id).\ - filter_by(deleted=False).\ - first() - if not result: - raise exception.NotFound(_('Instance %s not found') % (internal_id)) - - return result - - @require_admin_context def instance_get_all(context): session = get_session() diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3df561e7c1b4..2101297712d0 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -516,6 +516,12 @@ class SimpleDH(object): the openssl binary be installed on the system on which this is run, as it uses that to handle the encryption and decryption. If openssl is not available, a RuntimeError will be raised. + + Please note that nova already uses the M2Crypto library for most + cryptographic functions, and that it includes a Diffie-Hellman + implementation. However, that is a much more complex implementation, + and is not compatible with the DH algorithm that the agent uses. Hence + the need for this 'simple' version. """ def __init__(self, prime=None, base=None, secret=None): """You can specify the values for prime and base if you wish; diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index c9428e3a6877..3ea81474b4d4 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -276,7 +276,6 @@ class XenAPISession(object): error_info)) done.send_exception(self.XenAPI.Failure(error_info)) db.instance_action_create(context.get_admin_context(), action) - except self.XenAPI.Failure, exc: logging.warn(exc) done.send_exception(*sys.exc_info()) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index ab5b98d1cd20..70726bf6f8de 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -1,7 +1,8 @@ #!/usr/bin/env python -# Copyright (c) 2010 Citrix Systems, Inc. -# Copyright 2010 United States Government as represented by the +# Copyright (c) 2011 Citrix Systems, Inc. +# Copyright 2011 OpenStack LLC. +# Copyright 2011 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. # @@ -138,7 +139,8 @@ def _run_command(cmd): Otherwise, the output from stdout is returned. """ pipe = subprocess.PIPE - proc = subprocess.Popen([cmd], shell=True, stdin=pipe, stdout=pipe, stderr=pipe, close_fds=True) + proc = subprocess.Popen([cmd], shell=True, stdin=pipe, stdout=pipe, + stderr=pipe, close_fds=True) proc.wait() err = proc.stderr.read() if err: From 5d9ad54cc38283d0b946779f4235f54370b12489 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 7 Jan 2011 15:50:43 -0500 Subject: [PATCH 46/84] incorporated changes suggested by eday --- nova/api/openstack/servers.py | 9 +++------ nova/compute/api.py | 13 +++++++------ nova/compute/manager.py | 1 + 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index c71cb8562246..c97be86a2760 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -144,17 +144,14 @@ class Controller(wsgi.Controller): ctxt = req.environ['nova.context'] update_dict = {} - func = None if 'adminPass' in inst_dict['server']: update_dict['admin_pass'] = inst_dict['server']['adminPass'] - func = self.compute_api.set_admin_password - if 'name' in inst_dict['server']: - update_dict['display_name'] = inst_dict['server']['name'] - if func: try: - func(ctxt, id) + self.compute_api.set_admin_password(ctxt, id) except exception.TimeoutException, e: return exc.HTTPRequestTimeout() + if 'name' in inst_dict['server']: + update_dict['display_name'] = inst_dict['server']['name'] try: self.compute_api.update(ctxt, id, **update_dict) except exception.NotFound: diff --git a/nova/compute/api.py b/nova/compute/api.py index a894a0ce388f..d90b59de6985 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -239,8 +239,8 @@ class API(base.Base): host = instance['host'] if host: - self._cast_compute_message('terminate_instance', context, - instance_id) + self._cast_compute_message('snapshot_instance', context, + instance_id, host) else: self.db.instance_destroy(context, instance_id) @@ -268,12 +268,13 @@ class API(base.Base): project_id) return self.db.instance_get_all(context) - def _cast_compute_message(self, method, context, instance_id): + def _cast_compute_message(self, method, context, instance_id, host=None): """Generic handler for RPC calls to compute.""" - instance = self.get(context, instance_id) - host = instance['host'] + if not host: + instance = self.get(context, instance_id) + host = instance['host'] queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - kwargs = {'method': method, 'args': {'instance_id': instance['id']}} + kwargs = {'method': method, 'args': {'instance_id': instance_id}} rpc.cast(context, queue, kwargs) def snapshot(self, context, instance_id, name): diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 5d677b023f27..7d4a097bd0a8 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -285,6 +285,7 @@ class ComputeManager(manager.Manager): self.driver.snapshot(instance_ref, name) @exception.wrap_exception + @checks_instance_lock def set_admin_password(self, context, instance_id, new_pass=None): """Set the root/admin password for an instance on this server.""" context = context.elevated() From b8fc639af336630c56ce3807639a5e26c0d07982 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Fri, 7 Jan 2011 13:02:55 -0800 Subject: [PATCH 47/84] set the hostname factory in the service init --- nova/api/ec2/cloud.py | 7 ++++--- nova/compute/api.py | 22 +++++----------------- nova/tests/test_direct.py | 2 +- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 8e3091a68e1f..b16ab42bde7c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -93,8 +93,10 @@ class CloudController(object): def __init__(self): self.network_manager = utils.import_object(FLAGS.network_manager) self.image_service = utils.import_object(FLAGS.image_service) - self.compute_api = compute_api.ComputeAPI(self.network_manager, - self.image_service) + self.compute_api = compute_api.ComputeAPI( + network_manager=self.network_manager, + image_service=self.image_service, + hostname_factory=internal_id_to_ec2_id) self.setup() def __str__(self): @@ -807,7 +809,6 @@ class CloudController(object): key_name=kwargs.get('key_name'), user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), - hostname_format='ec2', availability_zone=kwargs.get('placement', {}).get( 'AvailabilityZone')) return self._format_run_instances(context, diff --git a/nova/compute/api.py b/nova/compute/api.py index 821fb06d5cf3..591d99271b0b 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -41,28 +41,18 @@ def id_to_default_hostname(internal_id): return str(internal_id) -def id_to_ec2_hostname(internal_id): - digits = [] - while internal_id != 0: - internal_id, remainder = divmod(internal_id, 36) - digits.append('0123456789abcdefghijklmnopqrstuvwxyz'[remainder]) - return "i-%s" % ''.join(reversed(digits)) - - -HOSTNAME_FORMATTERS = {'default': id_to_default_hostname, - 'ec2': id_to_ec2_hostname} - - class ComputeAPI(base.Base): """API for interacting with the compute manager.""" - def __init__(self, network_manager=None, image_service=None, **kwargs): + def __init__(self, network_manager=None, image_service=None, + hostname_factory=id_to_default_hostname, **kwargs): if not network_manager: network_manager = utils.import_object(FLAGS.network_manager) self.network_manager = network_manager if not image_service: image_service = utils.import_object(FLAGS.image_service) self.image_service = image_service + self.hostname_factory = hostname_factory super(ComputeAPI, self).__init__(**kwargs) def get_network_topic(self, context, instance_id): @@ -88,8 +78,7 @@ class ComputeAPI(base.Base): display_name='', description='', key_name=None, key_data=None, security_group='default', availability_zone=None, - user_data=None, - hostname_format='default'): + user_data=None): """Create the number of instances requested if quote and other arguments check out ok.""" @@ -160,7 +149,6 @@ class ComputeAPI(base.Base): elevated = context.elevated() instances = [] - generate_hostname = HOSTNAME_FORMATTERS[hostname_format] logging.debug(_("Going to run %s instances..."), num_instances) for num in range(num_instances): instance = dict(mac_address=utils.generate_mac(), @@ -179,7 +167,7 @@ class ComputeAPI(base.Base): security_group_id) # Set sane defaults if not specified - updates = dict(hostname=generate_hostname(internal_id)) + updates = dict(hostname=self.hostname_factory(internal_id)) if 'display_name' not in instance: updates['display_name'] = "Server %s" % internal_id diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index d73c64ce038f..78ad8ffed9c6 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -72,7 +72,7 @@ class DirectTestCase(test.TestCase): resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') - def test_req_params(self): + def test_post_params(self): req = webob.Request.blank('/fake/echo') req.environ['openstack.context'] = self.context req.method = 'POST' From 18b8d8307d0fc008f62dd8eeeedb351a954a3471 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 7 Jan 2011 15:51:28 -0600 Subject: [PATCH 48/84] removed a merge conflict line I missed before --- nova/compute/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 8b5646a54595..21b09e4439a1 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -37,7 +37,6 @@ terminating it. import datetime import random import string ->>>>>>> MERGE-SOURCE import functools from nova import exception From a0ec77b597713fd9a4be5bb7b892eba4ac53e625 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Fri, 7 Jan 2011 15:56:32 -0600 Subject: [PATCH 49/84] Reverted formatting change no longer necessary --- nova/api/openstack/servers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index d8a7ca161a19..764c843ac424 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -104,8 +104,7 @@ class Controller(wsgi.Controller): def show(self, req, id): """ Returns server details by server id """ try: - instance = self.compute_api.get( - req.environ['nova.context'], id) + instance = self.compute_api.get(req.environ['nova.context'], id) return _translate_detail_keys(instance) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) From 8fe01c087943ca9b46d25c84d4408b752461e6bd Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Fri, 7 Jan 2011 16:05:06 -0800 Subject: [PATCH 50/84] some small cleanups --- nova/compute/api.py | 6 +----- nova/tests/test_direct.py | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 65e772d810f2..ffef20cee656 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -253,11 +253,7 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" rv = self.db.instance_get_by_id(context, instance_id) - d = dict(rv.iteritems()) - # TODO(termie): this is ugly but required because the db layer returns - # models rather than dicts and the models support - # properties that are 'backreferences' - return d + return dict(rv.iteritems()) def get_all(self, context, project_id=None, reservation_id=None, fixed_ip=None): diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 5c6ecd903920..8a74b2296760 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -101,7 +101,3 @@ class DirectCloudTestCase(test_cloud.CloudTestCase): def tearDown(self): super(DirectCloudTestCase, self).tearDown() direct.ROUTES = {} - -if __name__ == '__main__': - import unittest - unittest.main() From d79600c1029ab91de8a81809df9efddc762351c0 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Fri, 7 Jan 2011 16:42:38 -0800 Subject: [PATCH 51/84] small cleanups --- nova/api/openstack/servers.py | 2 +- nova/db/sqlalchemy/models.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 0953b01ed49c..3f0fdc5753e2 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -49,7 +49,7 @@ def _translate_detail_keys(inst): power_state.SHUTOFF: 'active', power_state.CRASHED: 'error'} inst_dict = {} - print inst + mapped_keys = dict(status='state', imageId='image_id', flavorId='instance_type', name='display_name', id='id') diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 14cf7f617e1a..d730a0ebc38a 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -90,7 +90,9 @@ class NovaBase(object): setattr(self, k, v) def iteritems(self): - """Make the model object behave like a dict""" + """Make the model object behave like a dict. + + Includes attributes from joins.""" local = dict(self) joined = dict([(k, v) for k, v in self.__dict__.iteritems() if not k[0] == '_']) From bae57e82767b4877bae5c2dcb6fe052291d16b32 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Mon, 10 Jan 2011 15:33:10 -0600 Subject: [PATCH 52/84] Fixed issues raised by reviews --- nova/compute/api.py | 2 +- nova/virt/xenapi/vmops.py | 15 +++++++-------- .../xenserver/xenapi/etc/xapi.d/plugins/agent | 16 ---------------- 3 files changed, 8 insertions(+), 25 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index a20dc59cb6ab..10d7b67cfd09 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -297,7 +297,7 @@ class API(base.Base): host = instance['host'] if host: - self._cast_compute_message('snapshot_instance', context, + self._cast_compute_message('terminate_instance', context, instance_id, host) else: self.db.instance_destroy(context, instance_id) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 78f35e9a7f88..206970c35d52 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -139,17 +139,19 @@ class VMOps(object): return instance_or_vm else: # Must be the instance name - instance = instance_or_vm + instance_name = instance_or_vm except (AttributeError, KeyError): # Note the the KeyError will only happen with fakes.py # Not a string; must be an ID or a vm instance if isinstance(instance_or_vm, (int, long)): - instance = instance_or_vm + ctx = context.get_admin_context() + instance_obj = db.instance_get_by_id(ctx, instance_or_vm) + instance_name = instance_obj.name else: - instance = instance_or_vm.name - vm = VMHelper.lookup(self._session, instance) + instance_name = instance_or_vm.name + vm = VMHelper.lookup(self._session, instance_name) if vm is None: - raise Exception(_('Instance not present %s') % instance) + raise Exception(_('Instance not present %s') % instance_name) return vm def snapshot(self, instance, name): @@ -378,9 +380,6 @@ class VMOps(object): rec = self._session.get_xenapi().VM.get_record(vm) args = {'dom_id': rec['domid'], 'path': path} args.update(addl_args) - # If the 'testing_mode' attribute is set, add that to the args. - if getattr(self, 'testing_mode', False): - args['testing_mode'] = 'true' try: task = self._session.async_call_plugin(plugin, method, args) ret = self._session.wait_for_task(instance_id, task) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index 70726bf6f8de..82dd5466e958 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -39,8 +39,6 @@ configure_logging("xenstore") import xenstore AGENT_TIMEOUT = 30 -# Used for simulating an external agent for testing -PRETEND_SECRET = 11111 def jsonify(fnc): @@ -158,13 +156,6 @@ def key_init(self, arg_dict): pub = int(arg_dict["pub"]) arg_dict["value"] = json.dumps({"name": "keyinit", "value": pub}) request_id = arg_dict["id"] - if arg_dict.get("testing_mode"): - # Pretend! - pretend = SimpleDH(secret=PRETEND_SECRET) - shared = pretend.compute_shared(pub) - # Simulate the agent's response - ret = {"returncode": "D0", "message": pretend.get_public(), "shared": shared} - return ret arg_dict["path"] = "data/host/%s" % request_id xenstore.write_record(self, arg_dict) try: @@ -184,13 +175,6 @@ def password(self, arg_dict): """ pub = int(arg_dict["pub"]) enc_pass = arg_dict["enc_pass"] - if arg_dict.get("testing_mode"): - # Decrypt the password, and send it back to verify - pretend = SimpleDH(secret=PRETEND_SECRET) - pretend.compute_shared(pub) - pw = pretend.decrypt(enc_pass) - ret = {"returncode": "0", "message": pw} - return ret arg_dict["value"] = json.dumps({"name": "password", "value": enc_pass}) request_id = arg_dict["id"] arg_dict["path"] = "data/host/%s" % request_id From d91a06b4fea7e45fd2e9abe35803cd9deb5d8e92 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Tue, 11 Jan 2011 12:17:39 -0600 Subject: [PATCH 53/84] Removed unneeded SimpleDH code from agent plugin. Improved handling of plugin call failures. --- nova/virt/xenapi/vmops.py | 14 ++- .../xenserver/xenapi/etc/xapi.d/plugins/agent | 95 ------------------- 2 files changed, 12 insertions(+), 97 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 206970c35d52..c10943aa12e0 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -220,6 +220,9 @@ class VMOps(object): dh = SimpleDH() args = {'id': transaction_id, 'pub': str(dh.get_public())} resp = self._make_agent_call('key_init', instance, '', args) + if resp is None: + # No response from the agent + return resp_dict = json.loads(resp) # Successful return code from key_init is 'D0' if resp_dict['returncode'] != 'D0': @@ -232,6 +235,9 @@ class VMOps(object): # Send the encrypted password args['enc_pass'] = enc_pass resp = self._make_agent_call('password', instance, '', args) + if resp is None: + # No response from the agent + return resp_dict = json.loads(resp) # Successful return code from password is '0' if resp_dict['returncode'] != '0': @@ -384,12 +390,16 @@ class VMOps(object): task = self._session.async_call_plugin(plugin, method, args) ret = self._session.wait_for_task(instance_id, task) except self.XenAPI.Failure, e: + ret = None err_trace = e.details[-1] err_msg = err_trace.splitlines()[-1] + strargs = str(args) if 'TIMEOUT:' in err_msg: - raise exception.TimeoutException(err_msg) + LOG.error(_('TIMEOUT: The call to %(method)s timed out. ' + 'VM id=%(instance_id)s; args=%(strargs)s') % locals()) else: - raise RuntimeError("%s" % e.details[-1]) + LOG.error(_('The call to %(method)s returned an error: %(e)s. ' + 'VM id=%(instance_id)s; args=%(strargs)s') % locals()) return ret def add_to_xenstore(self, vm, path, key, value): diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index 82dd5466e958..12c3a19c8d0c 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -51,101 +51,6 @@ class TimeoutError(StandardError): pass -class SimpleDH(object): - """This class wraps all the functionality needed to implement - basic Diffie-Hellman-Merkle key exchange in Python. It features - intelligent defaults for the prime and base numbers needed for the - calculation, while allowing you to supply your own. It requires that - the openssl binary be installed on the system on which this is run, - as it uses that to handle the encryption and decryption. If openssl - is not available, a RuntimeError will be raised. - - Please note that nova already uses the M2Crypto library for most - cryptographic functions, and that it includes a Diffie-Hellman - implementation. However, that is a much more complex implementation, - and is not compatible with the DH algorithm that the agent uses. Hence - the need for this 'simple' version. - """ - def __init__(self, prime=None, base=None, secret=None): - """You can specify the values for prime and base if you wish; - otherwise, reasonable default values will be used. - """ - if prime is None: - self._prime = 162259276829213363391578010288127 - else: - self._prime = prime - if base is None: - self._base = 5 - else: - self._base = base - if secret is None: - self._secret = random.randint(5000, 15000) - else: - self._secret = secret - self._shared = self._public = None - - def get_public(self): - """Return the public key""" - self._public = (self._base ** self._secret) % self._prime - return self._public - - def compute_shared(self, other): - """Given the other end's public key, compute the - shared secret. - """ - self._shared = (other ** self._secret) % self._prime - return self._shared - - def _run_ssl(self, text, which): - """The encryption/decryption methods require running the openssl - installed on the system. This method abstracts out the common - code required. - """ - base_cmd = ("cat %(tmpfile)s | openssl enc -aes-128-cbc " - "-a -pass pass:%(shared)s -nosalt %(dec_flag)s") - if which.lower()[0] == "d": - dec_flag = " -d" - else: - dec_flag = "" - # Note: instead of using 'cat' and a tempfile, it is also - # possible to just 'echo' the value. However, we can not assume - # that the value is 'safe'; i.e., it may contain semi-colons, - # octothorpes, or other characters that would not be allowed - # in an 'echo' construct. - fd, tmpfile = tempfile.mkstemp() - os.close(fd) - file(tmpfile, "w").write(text) - shared = self._shared - cmd = base_cmd % locals() - try: - return _run_command(cmd) - except PluginError, e: - raise RuntimeError("OpenSSL error: %s" % e) - - def encrypt(self, text): - """Uses the shared key to encrypt the given text.""" - return self._run_ssl(text, "enc") - - def decrypt(self, text): - """Uses the shared key to decrypt the given text.""" - return self._run_ssl(text, "dec") - - -def _run_command(cmd): - """Abstracts out the basics of issuing system commands. If the command - returns anything in stderr, a PluginError is raised with that information. - Otherwise, the output from stdout is returned. - """ - pipe = subprocess.PIPE - proc = subprocess.Popen([cmd], shell=True, stdin=pipe, stdout=pipe, - stderr=pipe, close_fds=True) - proc.wait() - err = proc.stderr.read() - if err: - raise PluginError(err) - return proc.stdout.read() - - @jsonify def key_init(self, arg_dict): """Handles the Diffie-Hellman key exchange with the agent to From c14425541a1e77eb2049b94060bc0c4cd1df578f Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 12 Jan 2011 02:15:09 -0500 Subject: [PATCH 54/84] changed exception class --- nova/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index afe7422d9e54..888ebfd81c6a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -219,10 +219,13 @@ def get_my_linklocal(interface): if address[0] is not None: return address[0] else: - return None - except RuntimeError as ex: - logging.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex) - return 'fe00::' + return 'fe00::' + except IndexError as ex: + logging.warn(_("Couldn't parse command from get Link Local IP of %s :%s"), interface, ex) + except ProcessExecutionError as ex: + logging.warn(_("Couldn't execute command to get Link Local IP of %s :%s"), interface, ex) + except: + return 'fe00::' def to_global_ipv6(prefix, mac): From 1629dcf935a29c01d4e4ad509e33356daa93b051 Mon Sep 17 00:00:00 2001 From: Hisaharu Ishii Date: Wed, 12 Jan 2011 11:26:22 +0900 Subject: [PATCH 55/84] Fixed for pep8 Remove temporary debugging --- bin/nova-manage | 10 +++++----- nova/db/api.py | 3 +++ nova/db/sqlalchemy/api.py | 4 ++++ nova/utils.py | 1 - 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 04ecc8c10b75..7331186c7bfa 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -91,6 +91,7 @@ flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') + class VpnCommands(object): """Class for managing VPNs.""" @@ -432,11 +433,12 @@ class NetworkCommands(object): """Class for managing networks.""" def create(self, fixed_range=None, num_networks=None, - network_size=None, vlan_start=None, vpn_start=None,fixed_range_v6=None): + network_size=None, vlan_start=None, vpn_start=None, + fixed_range_v6=None): """Creates fixed ips for host by range arguments: [fixed_range=FLAG], [num_networks=FLAG], [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG],[fixed_range_v6=FLAG]""" + [vpn_start=FLAG], [fixed_range_v6=FLAG]""" if not fixed_range: fixed_range = FLAGS.fixed_range if not num_networks: @@ -453,9 +455,7 @@ class NetworkCommands(object): net_manager.create_networks(context.get_admin_context(), fixed_range, int(num_networks), int(network_size), int(vlan_start), - int(vpn_start),fixed_range_v6) - - + int(vpn_start), fixed_range_v6) class ServiceCommands(object): diff --git a/nova/db/api.py b/nova/db/api.py index 8684a3aef4ec..03e8004664cc 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -284,6 +284,7 @@ def fixed_ip_get_instance(context, address): """Get an instance for a fixed ip by address.""" return IMPL.fixed_ip_get_instance(context, address) + def fixed_ip_get_instance_v6(context, address): return IMPL.fixed_ip_get_instance_v6(context, address) @@ -345,6 +346,7 @@ def instance_get_fixed_address(context, instance_id): """Get the fixed ip address of an instance.""" return IMPL.instance_get_fixed_address(context, instance_id) + def instance_get_fixed_address_v6(context, instance_id): return IMPL.instance_get_fixed_address_v6(context, instance_id) @@ -543,6 +545,7 @@ def project_get_network(context, project_id, associate=True): return IMPL.project_get_network(context, project_id) + def project_get_network_v6(context, project_id): return IMPL.project_get_network_v6(context, project_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index ffc0ec221d73..7c3afa4adb45 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -585,6 +585,7 @@ def fixed_ip_get_instance(context, address): fixed_ip_ref = fixed_ip_get_by_address(context, address) return fixed_ip_ref.instance + @require_context def fixed_ip_get_instance_v6(context, address): session = get_session() @@ -801,6 +802,7 @@ def instance_get_fixed_address(context, instance_id): return None return instance_ref.fixed_ip['address'] + @require_context def instance_get_fixed_address_v6(context, instance_id): session = get_session() @@ -811,6 +813,7 @@ def instance_get_fixed_address_v6(context, instance_id): mac = instance_ref.mac_address return utils.to_global_ipv6(prefix, mac) + @require_context def instance_get_floating_address(context, instance_id): session = get_session() @@ -1150,6 +1153,7 @@ def project_get_network(context, project_id, associate=True): first() return result + @require_context def project_get_network_v6(context, project_id): return project_get_network(context, project_id) diff --git a/nova/utils.py b/nova/utils.py index 888ebfd81c6a..b08d5d620e4b 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -47,7 +47,6 @@ TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" def import_class(import_str): """Returns a class from a string including module and class""" mod_str, _sep, class_str = import_str.rpartition('.') - logging.debug(import_str) try: __import__(mod_str) return getattr(sys.modules[mod_str], class_str) From d368d182d7fa4b0f0cd9c7c5ad1e804b19365b26 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 12 Jan 2011 12:05:27 +0900 Subject: [PATCH 56/84] Add DescribeInstanceV6 for backward compatibility --- contrib/boto_v6/ec2/connection.py | 2 +- nova/api/ec2/cloud.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/contrib/boto_v6/ec2/connection.py b/contrib/boto_v6/ec2/connection.py index 151b76a556c8..23466e5d70c8 100644 --- a/contrib/boto_v6/ec2/connection.py +++ b/contrib/boto_v6/ec2/connection.py @@ -37,5 +37,5 @@ class EC2ConnectionV6(boto.ec2.EC2Connection): self.build_list_params(params, instance_ids, 'InstanceId') if filters: self.build_filter_params(params, filters) - return self.get_list('DescribeInstances', params, + return self.get_list('DescribeInstancesV6', params, [('item', ReservationV6)]) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 8c925ffeeca8..3d071782cfdc 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -643,15 +643,21 @@ class CloudController(object): def describe_instances(self, context, **kwargs): return self._format_describe_instances(context) + def describe_instances_v6(self, context, **kwargs): + return self._format_describe_instances(context) + def _format_describe_instances(self, context): return {'reservationSet': self._format_instances(context)} + def _format_describe_instances_v6(self, context): + return {'reservationSet': self._format_instances(context,None,True)} + def _format_run_instances(self, context, reservation_id): i = self._format_instances(context, reservation_id) assert len(i) == 1 return i[0] - def _format_instances(self, context, reservation_id=None): + def _format_instances(self, context, reservation_id=None,use_v6=False): reservations = {} if reservation_id: instances = db.instance_get_all_by_reservation(context, @@ -677,7 +683,7 @@ class CloudController(object): if instance['fixed_ip']['floating_ips']: fixed = instance['fixed_ip'] floating_addr = fixed['floating_ips'][0]['address'] - if instance['fixed_ip']['network'] and FLAGS.use_ipv6: + if instance['fixed_ip']['network'] and use_v6: i['dnsNameV6'] = utils.to_global_ipv6( instance['fixed_ip']['network']['cidr_v6'], instance['mac_address']) From 7a6b7c32ed25d1edc58b924ce5621dc0d8de9686 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 12 Jan 2011 03:50:09 -0500 Subject: [PATCH 57/84] Fixed bug --- nova/api/ec2/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 3d071782cfdc..d6f164b5c888 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -644,7 +644,7 @@ class CloudController(object): return self._format_describe_instances(context) def describe_instances_v6(self, context, **kwargs): - return self._format_describe_instances(context) + return self._format_describe_instances_v6(context) def _format_describe_instances(self, context): return {'reservationSet': self._format_instances(context)} From 04cd3241f442f1c6a9fd030ab47b4d15e79ec032 Mon Sep 17 00:00:00 2001 From: Hisaharu Ishii Date: Wed, 12 Jan 2011 18:37:18 +0900 Subject: [PATCH 58/84] Change command to get link local address Remove superfluous code --- nova/utils.py | 4 ++-- nova/virt/libvirt_conn.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 02bafc6c85bc..a13fa214c244 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -212,8 +212,8 @@ def get_my_ip(): def get_my_linklocal(interface): try: - if_str = execute("ifconfig %s" % interface) - condition = "\s+inet6\s+addr:\s+([0-9a-f:]+/\d+)\s+Scope:Link" + if_str = execute("ip -f inet6 -o addr show %s" % interface) + condition = "\s+inet6\s+([0-9a-f:]+/\d+)\s+scope:link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] if address[0] is not None: diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index c6827edbe9a6..e5f61f9aac09 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1024,7 +1024,6 @@ class NWFilterFirewall(FirewallDriver): security_group = db.security_group_get(context.get_admin_context(), security_group_id) rule_xml = "" - version = 4 v6protocol = {'tcp': 'tcp-ipv6', 'udp': 'udp-ipv6', 'icmp': 'icmpv6'} for rule in security_group.rules: rule_xml += "" From b4600b088b61a5653be9a93a0497c9d80916c8c0 Mon Sep 17 00:00:00 2001 From: MORITA Kazutaka Date: Wed, 12 Jan 2011 21:12:25 +0900 Subject: [PATCH 59/84] Check whether 'device_path' has ':' before splitting it --- nova/virt/libvirt_conn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 51c805c2f420..1ea3b0aa4917 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -224,14 +224,14 @@ class LibvirtConnection(object): virt_dom = self._conn.lookupByName(instance_name) mount_device = mountpoint.rpartition("/")[2] xml = '' - (protocol, vol_name) = device_path.split(':') if device_path.startswith('/dev/'): xml = """ """ % (device_path, mount_device) - elif vol_name != '': + elif device_path.find(':') >= 0: + (protocol, vol_name) = device_path.split(':', 1) xml = """ From a6a2a057d8a027781e4270c9abc4f815c67293ec Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Wed, 12 Jan 2011 10:12:18 -0500 Subject: [PATCH 60/84] Fixed syntax errors --- nova/api/ec2/cloud.py | 2 +- nova/utils.py | 2 +- nova/virt/libvirt_conn.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 72d7bcc9593d..c5c99f1323d8 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -610,7 +610,7 @@ class CloudController(object): def describe_instances_v6(self, context, **kwargs): kwargs['use_v6'] = True - return self._format_describe_instances_v6(context, **kwargs) + return self._format_describe_instances(context, **kwargs) def _format_describe_instances(self, context, **kwargs): return {'reservationSet': self._format_instances(context, **kwargs)} diff --git a/nova/utils.py b/nova/utils.py index 49344699ad64..09c9a5f895ed 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -218,7 +218,7 @@ def get_my_ip(): def get_my_linklocal(interface): try: if_str = execute("ip -f inet6 -o addr show %s" % interface) - condition = "\s+inet6\s+([0-9a-f:]+/\d+)\s+scope:link" + condition = "\s+inet6\s+([0-9a-f:]+/\d+)\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] if address[0] is not None: diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index ec17e4db056e..2631387102b9 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -940,8 +940,8 @@ class NWFilterFirewall(FirewallDriver): ['no-mac-spoofing', 'no-ip-spoofing', 'no-arp-spoofing', - 'allow-dhcp-server', - 'allow-ra-server'])) + 'allow-dhcp-server' + ])) self._define_filter(self.nova_base_ipv4_filter) self._define_filter(self.nova_base_ipv6_filter) self._define_filter(self.nova_dhcp_filter) @@ -1035,6 +1035,8 @@ class NWFilterFirewall(FirewallDriver): instance_secgroup_filter_children = ['nova-base-ipv4', 'nova-base-ipv6', 'nova-allow-dhcp-server'] + if FLAGS.use_ipv6: + instance_secgroup_filter_children += ['nova-allow-ra-server'] ctxt = context.get_admin_context() From b945fed7779bddf799aa4a180d44745052d2da8c Mon Sep 17 00:00:00 2001 From: Hisaharu Ishii Date: Wed, 12 Jan 2011 21:55:36 +0900 Subject: [PATCH 61/84] Support IPv6 firewall with IptablesFirewallDriver --- nova/db/sqlalchemy/api.py | 2 +- nova/virt/libvirt_conn.py | 66 ++++++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 3b3a881709db..2ca16283f04c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -798,7 +798,7 @@ def instance_get_fixed_address_v6(context, instance_id): session = get_session() with session.begin(): instance_ref = instance_get(context, instance_id, session=session) - network_ref = project_get_network(context, context.project_id) + network_ref = network_get_by_instance(context, instance_id) prefix = network_ref.cidr_v6 mac = instance_ref.mac_address return utils.to_global_ipv6(prefix, mac) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 2631387102b9..f2ffbf18065c 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1131,11 +1131,17 @@ class IptablesFirewallDriver(FirewallDriver): def apply_ruleset(self): current_filter, _ = self.execute('sudo iptables-save -t filter') current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines) + new_filter = self.modify_rules(current_lines, 4) self.execute('sudo iptables-restore', process_input='\n'.join(new_filter)) + if(FLAGS.use_ipv6): + current_filter, _ = self.execute('sudo ip6tables-save -t filter') + current_lines = current_filter.split('\n') + new_filter = self.modify_rules(current_lines, 6) + self.execute('sudo ip6tables-restore', + process_input='\n'.join(new_filter)) - def modify_rules(self, current_lines): + def modify_rules(self, current_lines, ip_version): ctxt = context.get_admin_context() # Remove any trace of nova rules. new_filter = filter(lambda l: 'nova-' not in l, current_lines) @@ -1149,8 +1155,8 @@ class IptablesFirewallDriver(FirewallDriver): if not new_filter[rules_index].startswith(':'): break - our_chains = [':nova-ipv4-fallback - [0:0]'] - our_rules = ['-A nova-ipv4-fallback -j DROP'] + our_chains = [':nova-fallback - [0:0]'] + our_rules = ['-A nova-fallback -j DROP'] our_chains += [':nova-local - [0:0]'] our_rules += ['-A FORWARD -j nova-local'] @@ -1160,7 +1166,10 @@ class IptablesFirewallDriver(FirewallDriver): # First, we add instance chains and rules for instance in self.instances: chain_name = self._instance_chain_name(instance) - ip_address = self._ip_for_instance(instance) + if(ip_version == 4): + ip_address = self._ip_for_instance(instance) + elif(ip_version == 6): + ip_address = self._ip_for_instance_v6(instance) our_chains += [':%s - [0:0]' % chain_name] @@ -1186,13 +1195,19 @@ class IptablesFirewallDriver(FirewallDriver): our_rules += ['-A %s -j %s' % (chain_name, sg_chain_name)] - # Allow DHCP responses - dhcp_server = self._dhcp_server_for_instance(instance) - our_rules += ['-A %s -s %s -p udp --sport 67 --dport 68' % - (chain_name, dhcp_server)] + if(ip_version == 4): + # Allow DHCP responses + dhcp_server = self._dhcp_server_for_instance(instance) + our_rules += ['-A %s -s %s -p udp --sport 67 --dport 68' % + (chain_name, dhcp_server)] + elif(ip_version == 6): + # Allow RA responses + ra_server = self._ra_server_for_instance(instance) + our_rules += ['-A %s -s %s -p icmpv6' % + (chain_name, ra_server)] # If nothing matches, jump to the fallback chain - our_rules += ['-A %s -j nova-ipv4-fallback' % (chain_name,)] + our_rules += ['-A %s -j nova-fallback' % (chain_name,)] # then, security group chains and rules for security_group in security_groups: @@ -1205,15 +1220,22 @@ class IptablesFirewallDriver(FirewallDriver): for rule in rules: logging.info('%r', rule) - args = ['-A', chain_name, '-p', rule.protocol] - if rule.cidr: - args += ['-s', rule.cidr] - else: + if not rule.cidr: # Eventually, a mechanism to grant access for security # groups will turn up here. It'll use ipsets. continue + version = _get_ip_version(rule.cidr) + if version != ip_version: + continue + + protocol = rule.protocol + if version == 6 and rule.protocol == 'icmp': + protocol = 'icmpv6' + + args = ['-A', chain_name, '-p', protocol, '-s', rule.cidr] + if rule.protocol in ['udp', 'tcp']: if rule.from_port == rule.to_port: args += ['--dport', '%s' % (rule.from_port,)] @@ -1233,7 +1255,12 @@ class IptablesFirewallDriver(FirewallDriver): icmp_type_arg += '/%s' % icmp_code if icmp_type_arg: - args += ['-m', 'icmp', '--icmp-type', icmp_type_arg] + if(ip_version == 4): + args += ['-m', 'icmp', '--icmp-type', + icmp_type_arg] + elif(ip_version == 6): + args += ['-m', 'icmp6', '--icmpv6-type', + icmp_type_arg] args += ['-j ACCEPT'] our_rules += [' '.join(args)] @@ -1259,7 +1286,16 @@ class IptablesFirewallDriver(FirewallDriver): return db.instance_get_fixed_address(context.get_admin_context(), instance['id']) + def _ip_for_instance_v6(self, instance): + return db.instance_get_fixed_address_v6(context.get_admin_context(), + instance['id']) + def _dhcp_server_for_instance(self, instance): network = db.project_get_network(context.get_admin_context(), instance['project_id']) return network['gateway'] + + def _ra_server_for_instance(self, instance): + network = db.project_get_network(context.get_admin_context(), + instance['project_id']) + return network['ra_server'] From df0be0318cf22d250bdf9abdd9ed3b91bb83f0ea Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Thu, 13 Jan 2011 09:10:44 +0900 Subject: [PATCH 62/84] fixed method signature of modify_rules fixed unit_test for ipv6 --- nova/tests/test_network.py | 17 +++++++++-------- nova/virt/libvirt_conn.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 407dbc769a7b..00f9323f39ee 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -99,23 +99,24 @@ class NetworkTestCase(test.TestCase): def test_private_ipv6(self): """Make sure ipv6 is OK""" if FLAGS.use_ipv6: - instance_ref = self._create_instance(1) + instance_ref = self._create_instance(0) + address = self._create_address(0, instance_ref['id']) network_ref = db.project_get_network( - self.context, + context.get_admin_context(), self.context.project_id) address_v6 = db.instance_get_fixed_address_v6( - self.context, - instance_ref['id']) + context.get_admin_context(), + instance_ref['id']) self.assertEqual(instance_ref['mac_address'], utils.to_mac(address_v6)) instance_ref2 = db.fixed_ip_get_instance_v6( - self.context, - address_v6) + context.get_admin_context(), + address_v6) self.assertEqual(instance_ref['id'], instance_ref2['id']) self.assertEqual(address_v6, utils.to_global_ipv6( - network_ref['cidr_v6'], - instance_ref['mac_address'])) + network_ref['cidr_v6'], + instance_ref['mac_address'])) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index f2ffbf18065c..191292a9defd 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1141,7 +1141,7 @@ class IptablesFirewallDriver(FirewallDriver): self.execute('sudo ip6tables-restore', process_input='\n'.join(new_filter)) - def modify_rules(self, current_lines, ip_version): + def modify_rules(self, current_lines, ip_version=4): ctxt = context.get_admin_context() # Remove any trace of nova rules. new_filter = filter(lambda l: 'nova-' not in l, current_lines) From 3419feff16e1974aa353188eee11609fc786148d Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 12 Jan 2011 19:38:27 -0800 Subject: [PATCH 63/84] make sure get_all returns --- nova/compute/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index ffef20cee656..630754395203 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -261,8 +261,8 @@ class API(base.Base): given parameters. If there is no filter and the context is an admin, it will retreive all instances in the system.""" if reservation_id is not None: - rv = self.db.instance_get_all_by_reservation(context, - reservation_id) + return self.db.instance_get_all_by_reservation(context, + reservation_id) if fixed_ip is not None: return self.db.fixed_ip_get_instance(context, fixed_ip) if project_id or not context.is_admin: From a5026320b4ae14f0171ee450fe79ea687ab5647a Mon Sep 17 00:00:00 2001 From: Koji Iida Date: Thu, 13 Jan 2011 15:58:05 +0900 Subject: [PATCH 64/84] Fixed missing _(). Fixed to follow logging to LOG changes. Fixed merge miss (get_fixed_ip was moved away). Update some missing comments. --- nova/network/linux_net.py | 14 +++++++------- nova/tests/test_api.py | 1 - nova/utils.py | 17 ++--------------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 50256db2d391..6779316036bd 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -292,12 +292,12 @@ interface %s """ % (network_ref['bridge'], network_ref['cidr_v6']) f.write(conf_str) - # Make sure dnsmasq can actually read it (it setuid()s to "nobody") + # Make sure radvd can actually read it (it setuid()s to "nobody") os.chmod(conffile, 0644) pid = _ra_pid_for(network_ref['bridge']) - # if dnsmasq is already running, then tell it to reload + # if radvd is already running, then tell it to reload if pid: out, _err = _execute('cat /proc/%d/cmdline' % pid, check_exit_code=False) @@ -306,9 +306,9 @@ interface %s _execute('sudo kill -HUP %d' % pid) return except Exception as exc: # pylint: disable-msg=W0703 - logging.debug("Hupping radvd threw %s", exc) + LOG.debug(_("Hupping radvd threw %s"), exc) else: - logging.debug("Pid %d is stale, relaunching radvd", pid) + LOG.debug(_("Pid %d is stale, relaunching radvd"), pid) command = _ra_cmd(network_ref) _execute(command) db.network_update(context, network_id, @@ -378,7 +378,7 @@ def _dnsmasq_cmd(net): def _ra_cmd(net): - """Builds dnsmasq command""" + """Builds radvd command""" cmd = ['sudo -E radvd', # ' -u nobody', ' -C %s' % _ra_file(net['bridge'], 'conf'), @@ -408,7 +408,7 @@ def _dhcp_file(bridge, kind): def _ra_file(bridge, kind): - """Return path to a pid, leases or conf file for a bridge""" + """Return path to a pid or conf file for a bridge""" if not os.path.exists(FLAGS.networks_path): os.makedirs(FLAGS.networks_path) @@ -433,7 +433,7 @@ def _dnsmasq_pid_for(bridge): def _ra_pid_for(bridge): - """Returns the pid for prior dnsmasq instance for a bridge + """Returns the pid for prior radvd instance for a bridge Returns None if no pid file exists diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index a508235c43bc..d22d7beb1006 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -24,7 +24,6 @@ import httplib import random import StringIO import webob -import logging from nova import context from nova import flags diff --git a/nova/utils.py b/nova/utils.py index 09c9a5f895ed..27589c30ce11 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -202,19 +202,6 @@ def last_octet(address): return int(address.split(".")[-1]) -def get_my_ip(): - """Returns the actual ip of the local machine.""" - try: - csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - csock.connect(('8.8.8.8', 80)) - (addr, port) = csock.getsockname() - csock.close() - return addr - except socket.gaierror as ex: - logging.warn(_("Couldn't get IP, using 127.0.0.1 %s"), ex) - return "127.0.0.1" - - def get_my_linklocal(interface): try: if_str = execute("ip -f inet6 -o addr show %s" % interface) @@ -226,9 +213,9 @@ def get_my_linklocal(interface): else: return 'fe00::' except IndexError as ex: - logging.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex) + LOG.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex) except ProcessExecutionError as ex: - logging.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex) + LOG.warn(_("Couldn't get Link Local IP of %s :%s"), interface, ex) except: return 'fe00::' From 515b4fb9ec53f1fbcfc8e502a51e52f67706cc40 Mon Sep 17 00:00:00 2001 From: Hisaharu Ishii Date: Thu, 13 Jan 2011 16:59:29 +0900 Subject: [PATCH 65/84] Fixed Authors --- Authors | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Authors b/Authors index 7bcd5981f34f..479b796ba1af 100644 --- a/Authors +++ b/Authors @@ -45,7 +45,7 @@ Trey Morris Vishvananda Ishaya Youcef Laribi Zhixue Wu -Hisaharu Ishii -Koji Iida +Hisaharu Ishii +Koji Iida Nachi Ueno From 1c694e9093c627bd50b35e9fb0ae11adf315a154 Mon Sep 17 00:00:00 2001 From: MORITA Kazutaka Date: Thu, 13 Jan 2011 20:59:02 +0900 Subject: [PATCH 66/84] Revert r510 and r512 because Josh had already done the same work --- nova/virt/libvirt_conn.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 1ea3b0aa4917..00edfbdc8ddd 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -223,26 +223,11 @@ class LibvirtConnection(object): def attach_volume(self, instance_name, device_path, mountpoint): virt_dom = self._conn.lookupByName(instance_name) mount_device = mountpoint.rpartition("/")[2] - xml = '' - if device_path.startswith('/dev/'): - xml = """ - - - - """ % (device_path, mount_device) - elif device_path.find(':') >= 0: - (protocol, vol_name) = device_path.split(':', 1) - xml = """ - - - - """ % (protocol, vol_name, mount_device) - else: - xml = """ - - - - """ % (device_path, mount_device) + xml = """ + + + + """ % (device_path, mount_device) virt_dom.attachDevice(xml) def _get_disk_xml(self, xml, device): From 7af8f5ac5fc02abe79dec3cf3651b6f0a9deb78c Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 13 Jan 2011 10:51:31 -0600 Subject: [PATCH 67/84] Minor code cleanups --- nova/compute/manager.py | 2 +- nova/tests/test_xenapi.py | 4 ---- nova/virt/xenapi/vmops.py | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 267beca45f6e..613ee45f6310 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -125,7 +125,7 @@ class ComputeManager(manager.Manager): # FIXME(ja): include other fields from state? instance_ref = self.db.instance_get(context, instance_id) try: - info = self.driver.get_info(instance_ref) + info = self.driver.get_info(instance_ref['name']) state = info['state'] except exception.NotFound: state = power_state.NOSTATE diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index a03d616ad8a4..93afe9ce1aae 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -265,10 +265,6 @@ class XenAPIVMTestCase(test.TestCase): return instance - - - - class XenAPIDiffieHellmanTestCase(test.TestCase): """ Unit tests for Diffie-Hellman code diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6c6d25709adf..6e359ef82ba8 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -22,7 +22,6 @@ Management class for VM-related functions (spawn, reboot, etc). import json import M2Crypto import os -import random import subprocess import tempfile import uuid From 22b21cde84f200f6fd45ba5f2cfcb6a54e595f1b Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 13 Jan 2011 10:52:28 -0600 Subject: [PATCH 68/84] Minor code cleanups --- nova/tests/test_xenapi.py | 2 +- nova/virt/xenapi/vmops.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 93afe9ce1aae..261ee0fde901 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -286,6 +286,6 @@ class XenAPIDiffieHellmanTestCase(test.TestCase): enc = self.alice.encrypt(msg) dec = self.bob.decrypt(enc) self.assertEquals(dec, msg) - + def tearDown(self): super(XenAPIDiffieHellmanTestCase, self).tearDown() diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6e359ef82ba8..6c6d25709adf 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -22,6 +22,7 @@ Management class for VM-related functions (spawn, reboot, etc). import json import M2Crypto import os +import random import subprocess import tempfile import uuid From 01a1ad3d2cdf61c73ca3ab7aa14e82f0e4450103 Mon Sep 17 00:00:00 2001 From: Ed Leafe Date: Thu, 13 Jan 2011 10:53:13 -0600 Subject: [PATCH 69/84] Minor code cleanups --- nova/virt/xenapi/vmops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6c6d25709adf..6e359ef82ba8 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -22,7 +22,6 @@ Management class for VM-related functions (spawn, reboot, etc). import json import M2Crypto import os -import random import subprocess import tempfile import uuid From 2d642f64dcf0bcabf739aa1373eb2794112f9be4 Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Fri, 14 Jan 2011 06:18:16 +0900 Subject: [PATCH 70/84] Added netaddr for pip-requires --- tools/pip-requires | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/pip-requires b/tools/pip-requires index 341043114832..3aa76c24deb6 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -25,3 +25,4 @@ bzr Twisted>=10.1.0 PasteDeploy paste +netaddr From bc0c5ba5026610013759fa731d21e2287e3d709a Mon Sep 17 00:00:00 2001 From: Nachi Ueno Date: Fri, 14 Jan 2011 06:25:41 +0900 Subject: [PATCH 71/84] Moved commands which needs sudo to nova.sh --- contrib/nova.sh | 3 +++ nova/network/linux_net.py | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/contrib/nova.sh b/contrib/nova.sh index 3a895552627c..1f36d7f89f71 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -89,6 +89,9 @@ if [ "$CMD" == "install" ]; then #For IPV6 sudo apt-get install -y python-netaddr sudo apt-get install -y radvd +#(Nati) Note that this configuration is only needed for nova-network node. + sudo bash -c "echo 1 > /proc/sys/net/ipv6/conf/all/forwarding" + sudo bash -c "echo 0 > /proc/sys/net/ipv6/conf/all/accept_ra" if [ "$USE_MYSQL" == 1 ]; then cat < /proc/sys/net/ipv6/conf/all/forwarding"') - _execute('sudo bash -c ' + - '"echo 0 > /proc/sys/net/ipv6/conf/all/accept_ra"') def bind_floating_ip(floating_ip, check_exit_code=True): From 914b0554a092d2b38f292942dc4d7ddea5d99b9a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 13 Jan 2011 15:13:31 -0800 Subject: [PATCH 72/84] Modified per sorens review. Moved disk.py Removed disk.partition Changed docstrings Use pid to find nbd devices --- nova/{compute => virt}/disk.py | 89 ++++++---------------------------- nova/virt/libvirt_conn.py | 10 ++-- 2 files changed, 18 insertions(+), 81 deletions(-) rename nova/{compute => virt}/disk.py (66%) diff --git a/nova/compute/disk.py b/nova/virt/disk.py similarity index 66% rename from nova/compute/disk.py rename to nova/virt/disk.py index b9c34750d442..c5565abfaf8c 100644 --- a/nova/compute/disk.py +++ b/nova/virt/disk.py @@ -40,74 +40,6 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, 'block_size to use for dd') -def partition(infile, outfile, local_bytes=0, resize=True, local_type='ext2'): - """ - Turns a partition (infile) into a bootable drive image (outfile). - - The first 63 sectors (0-62) of the resulting image is a master boot record. - Infile becomes the first primary partition. - If local bytes is specified, a second primary partition is created and - formatted as ext2. - - :: - - In the diagram below, dashes represent drive sectors. - +-----+------. . .-------+------. . .------+ - | 0 a| b c|d e| - +-----+------. . .-------+------. . .------+ - | mbr | primary partiton | local partition | - +-----+------. . .-------+------. . .------+ - - """ - sector_size = 512 - file_size = os.path.getsize(infile) - if resize and file_size < FLAGS.minimum_root_size: - last_sector = FLAGS.minimum_root_size / sector_size - 1 - utils.execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (infile, last_sector, sector_size)) - utils.execute('e2fsck -fp %s' % infile, check_exit_code=False) - utils.execute('resize2fs %s' % infile) - file_size = FLAGS.minimum_root_size - elif file_size % sector_size != 0: - LOG.warn(_("Input partition size not evenly divisible by" - " sector size: %d / %d"), file_size, sector_size) - primary_sectors = file_size / sector_size - if local_bytes % sector_size != 0: - LOG.warn(_("Bytes for local storage not evenly divisible" - " by sector size: %d / %d"), local_bytes, sector_size) - local_sectors = local_bytes / sector_size - - mbr_last = 62 # a - primary_first = mbr_last + 1 # b - primary_last = primary_first + primary_sectors - 1 # c - local_first = primary_last + 1 # d - local_last = local_first + local_sectors - 1 # e - last_sector = local_last # e - - # create an empty file - utils.execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (outfile, mbr_last, sector_size)) - - # make mbr partition - utils.execute('parted --script %s mklabel msdos' % outfile) - - # append primary file - utils.execute('dd if=%s of=%s bs=%s conv=notrunc,fsync oflag=append' - % (infile, outfile, FLAGS.block_size)) - - # make primary partition - utils.execute('parted --script %s mkpart primary %ds %ds' - % (outfile, primary_first, primary_last)) - - if local_bytes > 0: - # make the file bigger - utils.execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (outfile, last_sector, sector_size)) - # make and format local partition - utils.execute('parted --script %s mkpartfs primary %s %ds %ds' - % (outfile, local_type, local_first, local_last)) - - def extend(image, size): """Increase image to size""" file_size = os.path.getsize(image) @@ -185,8 +117,11 @@ def _link_device(image, nbd): utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) # NOTE(vish): this forks into another process, so give it a chance # to set up before continuuing - time.sleep(1) - return device + for i in xrange(10): + if os.path.exists("/sys/block/%s/pid" % os.path.basename(device)): + return device + time.sleep(1) + raise exception.Error(_('nbd device %s did not show up') % device) else: out, err = utils.execute('sudo losetup --find --show %s' % image) if err: @@ -208,12 +143,16 @@ _DEVICES = ['/dev/nbd%s' % i for i in xrange(16)] def _allocate_device(): - # NOTE(vish): This assumes no other processes are using nbd devices. - # It will race cause a race condition if multiple + # NOTE(vish): This assumes no other processes are allocating nbd devices. + # It may race cause a race condition if multiple # workers are running on a given machine. - if not _DEVICES: - raise exception.Error(_('No free nbd devices')) - return _DEVICES.pop() + while True: + if not _DEVICES: + raise exception.Error(_('No free nbd devices')) + device = _DEVICES.pop() + if not os.path.exists("/sys/block/%s/pid" % os.path.basename(device)): + break + return device def _free_device(device): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 760bfef78a80..c77d25bfdae0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -58,9 +58,9 @@ from nova import log as logging from nova import utils #from nova.api import context from nova.auth import manager -from nova.compute import disk from nova.compute import instance_types from nova.compute import power_state +from nova.virt import disk from nova.virt import images libvirt = None @@ -495,7 +495,7 @@ class LibvirtConnection(object): return {'token': token, 'host': host, 'port': port} def _cache_image(self, fn, target, fname, cow=False, *args, **kwargs): - """Wrapper to cache a method that creates an image. + """Wrapper for a method that creates an image that caches the image. This wrapper will save the image into a common store and create a copy for use by the hypervisor. @@ -504,7 +504,7 @@ class LibvirtConnection(object): where the image will be saved. fname is used as the filename of the base image. The filename needs - to be fname to a given image. + to be unique to a given image. If cow is True, it will make a CoW image instead of a copy. """ @@ -531,9 +531,7 @@ class LibvirtConnection(object): def _create_local(self, target, local_gb): """Create a blank image of specified size""" - last_mb = local_gb * 1024 - 1 - utils.execute('dd if=/dev/zero of=%s bs=1M count=1 ' - 'seek=%s' % (target, last_mb)) + utils.execute('truncate %s -s %dG' % (target, local_gb)) # TODO(vish): should we format disk by default? def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): From 4ff82fe82729c46e64242a3a3a8aea1aff6ffeb1 Mon Sep 17 00:00:00 2001 From: Hisaharu Ishii Date: Fri, 14 Jan 2011 11:44:35 +0900 Subject: [PATCH 73/84] sort Authors --- Authors | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Authors b/Authors index 479b796ba1af..74736562e6f8 100644 --- a/Authors +++ b/Authors @@ -1,11 +1,12 @@ + Andy Smith Anne Gentle Anthony Young Antony Messerli Armando Migliaccio Chiradeep Vittal -Chris Behrens Chmouel Boudjnah +Chris Behrens Cory Wright David Pravec Dean Troyer @@ -14,6 +15,7 @@ Ed Leafe Eldar Nugaev Eric Day Ewan Mellor +Hisaharu Ishii Hisaki Ohara Ilya Alekseyev Jay Pipes @@ -26,11 +28,13 @@ Josh Kearney Joshua McKenty Justin Santa Barbara Ken Pepple +Koji Iida Lorin Hochstein Matt Dietz Michael Gundlach Monsyne Dragon Monty Taylor +Nachi Ueno Paul Voccio Rick Clark Rick Harris @@ -45,7 +49,3 @@ Trey Morris Vishvananda Ishaya Youcef Laribi Zhixue Wu -Hisaharu Ishii -Koji Iida -Nachi Ueno - From 25ada0ee2864ead19df82abf9419f956c0c39b2a Mon Sep 17 00:00:00 2001 From: MORITA Kazutaka Date: Fri, 14 Jan 2011 12:25:34 +0900 Subject: [PATCH 74/84] Update Authors --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 0c49f76a200d..3c25a8bb145f 100644 --- a/Authors +++ b/Authors @@ -25,6 +25,7 @@ Josh Durgin Josh Kearney Joshua McKenty Justin Santa Barbara +MORITA Kazutaka Ken Pepple Lorin Hochstein Matt Dietz From 600e397515ec425958a24c35a4440d85bd47133a Mon Sep 17 00:00:00 2001 From: MORITA Kazutaka Date: Fri, 14 Jan 2011 12:46:10 +0900 Subject: [PATCH 75/84] Sort Authors --- Authors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Authors b/Authors index 3c25a8bb145f..1b8cee2a1ae7 100644 --- a/Authors +++ b/Authors @@ -25,13 +25,13 @@ Josh Durgin Josh Kearney Joshua McKenty Justin Santa Barbara -MORITA Kazutaka Ken Pepple Lorin Hochstein Matt Dietz Michael Gundlach Monsyne Dragon Monty Taylor +MORITA Kazutaka Paul Voccio Rick Clark Rick Harris From cf0e5bd3eeb6b175b53df6ae0a0ef8957ec7ba13 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 14 Jan 2011 11:24:45 -0600 Subject: [PATCH 76/84] Create and use a generic handler for RPC calls --- nova/compute/api.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 90273da3600e..40b9e33e8acf 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -331,7 +331,7 @@ class API(base.Base): return self.db.instance_get_all(context) def _cast_compute_message(self, method, context, instance_id, host=None): - """Generic handler for RPC calls to compute.""" + """Generic handler for RPC casts to compute.""" if not host: instance = self.get(context, instance_id) host = instance['host'] @@ -339,6 +339,15 @@ class API(base.Base): kwargs = {'method': method, 'args': {'instance_id': instance_id}} rpc.cast(context, queue, kwargs) + def _call_compute_message(self, method, context, instance_id, host=None): + """Generic handler for RPC calls to compute.""" + if not host: + instance = self.get(context, instance_id) + host = instance["host"] + queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) + kwargs = {"method": method, "args": {"instance_id": instance_id}} + return rpc.call(context, queue, kwargs) + def snapshot(self, context, instance_id, name): """Snapshot the given instance.""" self._cast_compute_message('snapshot_instance', context, instance_id) @@ -357,7 +366,10 @@ class API(base.Base): def get_diagnostics(self, context, instance_id): """Retrieve diagnostics for the given instance.""" - self._cast_compute_message('get_diagnostics', context, instance_id) + return self._call_compute_message( + "get_diagnostics", + context, + instance_id) def get_actions(self, context, instance_id): """Retrieve actions for the given instance.""" From 76e875476848ee7f4aa483f65484903115e2bb49 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 14 Jan 2011 10:25:44 -0800 Subject: [PATCH 77/84] import re, remove extra call in cloud.py. Move get_console_output to compute_api --- nova/api/ec2/cloud.py | 17 +---------------- nova/compute/api.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 630aaeaf2b1e..4adcb7415456 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -130,15 +130,6 @@ class CloudController(object): result[key] = [line] return result - def _trigger_refresh_security_group(self, context, security_group): - nodes = set([instance['host'] for instance in security_group.instances - if instance['host'] is not None]) - for node in nodes: - rpc.cast(context, - '%s.%s' % (FLAGS.compute_topic, node), - {"method": "refresh_security_group", - "args": {"security_group_id": security_group.id}}) - def _get_availability_zone_by_host(self, context, host): services = db.service_get_all_by_host(context, host) if len(services) > 0: @@ -522,13 +513,7 @@ class CloudController(object): # instance_id is passed in as a list of instances ec2_id = instance_id[0] instance_id = ec2_id_to_id(ec2_id) - instance_ref = self.compute_api.get(context, instance_id) - output = rpc.call(context, - '%s.%s' % (FLAGS.compute_topic, - instance_ref['host']), - {"method": "get_console_output", - "args": {"instance_id": instance_ref['id']}}) - + output = self.compute_api.get_console_output(context, instance_id) now = datetime.datetime.utcnow() return {"InstanceId": ec2_id, "Timestamp": now, diff --git a/nova/compute/api.py b/nova/compute/api.py index 90273da3600e..e6cfa459eab5 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -21,6 +21,7 @@ Handles all requests relating to instances (guest vms). """ import datetime +import re import time from nova import db @@ -385,23 +386,23 @@ class API(base.Base): def get_ajax_console(self, context, instance_id): """Get a url to an AJAX Console""" - instance = self.get(context, instance_id) - - output = rpc.call(context, - '%s.%s' % (FLAGS.compute_topic, - instance['host']), - {'method': 'get_ajax_console', - 'args': {'instance_id': instance['id']}}) - + output = self._call_compute_message('get_ajax_console', + context, + instance_id) rpc.cast(context, '%s' % FLAGS.ajax_console_proxy_topic, {'method': 'authorize_ajax_console', 'args': {'token': output['token'], 'host': output['host'], 'port': output['port']}}) - return {'url': '%s?token=%s' % (FLAGS.ajax_console_proxy_url, output['token'])} + def get_console_output(self, context, instance_id): + """Get console output for an an instance""" + return self._call_compute_message('get_console_output', + context, + instance_id) + def lock(self, context, instance_id): """lock the instance with instance_id""" self._cast_compute_message('lock_instance', context, instance_id) From e0dcd52b98de4bfe9843b148decf22526713dea2 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 14 Jan 2011 11:00:47 -0800 Subject: [PATCH 78/84] remove TrialTestCase again and fix merge issues --- nova/test.py | 99 +---------------------------------- nova/tests/test_api.py | 4 +- nova/tests/test_log.py | 8 +-- nova/tests/test_middleware.py | 2 +- nova/tests/test_twistd.py | 2 +- 5 files changed, 10 insertions(+), 105 deletions(-) diff --git a/nova/test.py b/nova/test.py index 5922e4b1ccab..a26b85b04ab3 100644 --- a/nova/test.py +++ b/nova/test.py @@ -23,8 +23,6 @@ and some black magic for inline callbacks. """ import datetime -import sys -import time import unittest import mox @@ -38,7 +36,6 @@ from nova import fakerabbit from nova import flags from nova import rpc from nova.network import manager as network_manager -from nova.tests import fake_flags FLAGS = flags.FLAGS @@ -74,7 +71,8 @@ class TestCase(unittest.TestCase): FLAGS.fixed_range, 5, 16, FLAGS.vlan_start, - FLAGS.vpn_start) + FLAGS.vpn_start, + FLAGS.fixed_range_v6) # emulate some of the mox stuff, we can't use the metaclass # because it screws with our generators @@ -139,96 +137,3 @@ class TestCase(unittest.TestCase): _wrapped.func_name = self.originalAttach.func_name rpc.Consumer.attach_to_eventlet = _wrapped - - -class TrialTestCase(trial_unittest.TestCase): - """Test case base class for all unit tests""" - def setUp(self): - """Run before each test method to initialize test environment""" - super(TrialTestCase, self).setUp() - # NOTE(vish): We need a better method for creating fixtures for tests - # now that we have some required db setup for the system - # to work properly. - self.start = datetime.datetime.utcnow() - ctxt = context.get_admin_context() - if db.network_count(ctxt) != 5: - network_manager.VlanManager().create_networks(ctxt, - FLAGS.fixed_range, - 5, 16, - FLAGS.vlan_start, - FLAGS.vpn_start, - FLAGS.fixed_range_v6) - - # emulate some of the mox stuff, we can't use the metaclass - # because it screws with our generators - self.mox = mox.Mox() - self.stubs = stubout.StubOutForTesting() - self.flag_overrides = {} - self.injected = [] - self._original_flags = FLAGS.FlagValuesDict() - - def tearDown(self): - """Runs after each test method to finalize/tear down test - environment.""" - try: - self.mox.UnsetStubs() - self.stubs.UnsetAll() - self.stubs.SmartUnsetAll() - self.mox.VerifyAll() - # NOTE(vish): Clean up any ips associated during the test. - ctxt = context.get_admin_context() - db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, - self.start) - db.network_disassociate_all(ctxt) - for x in self.injected: - try: - x.stop() - except AssertionError: - pass - - if FLAGS.fake_rabbit: - fakerabbit.reset_all() - - db.security_group_destroy_all(ctxt) - super(TrialTestCase, self).tearDown() - finally: - self.reset_flags() - - def flags(self, **kw): - """Override flag variables for a test""" - for k, v in kw.iteritems(): - if k in self.flag_overrides: - self.reset_flags() - raise Exception( - 'trying to override already overriden flag: %s' % k) - self.flag_overrides[k] = getattr(FLAGS, k) - setattr(FLAGS, k, v) - - def reset_flags(self): - """Resets all flag variables for the test. Runs after each test""" - FLAGS.Reset() - for k, v in self._original_flags.iteritems(): - setattr(FLAGS, k, v) - - def run(self, result=None): - test_method = getattr(self, self._testMethodName) - setattr(self, - self._testMethodName, - self._maybeInlineCallbacks(test_method, result)) - rv = super(TrialTestCase, self).run(result) - setattr(self, self._testMethodName, test_method) - return rv - - def _maybeInlineCallbacks(self, func, result): - def _wrapped(): - g = func() - if isinstance(g, defer.Deferred): - return g - if not hasattr(g, 'send'): - return defer.succeed(g) - - inlined = defer.inlineCallbacks(func) - d = inlined() - return d - _wrapped.func_name = func.func_name - return _wrapped diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index d22d7beb1006..17789c25c77b 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -79,7 +79,7 @@ class FakeHttplibConnection(object): pass -class XmlConversionTestCase(test.TrialTestCase): +class XmlConversionTestCase(test.TestCase): """Unit test api xml conversion""" def test_number_conversion(self): conv = apirequest._try_convert @@ -96,7 +96,7 @@ class XmlConversionTestCase(test.TrialTestCase): self.assertEqual(conv('-0'), 0) -class ApiEc2TestCase(test.TrialTestCase): +class ApiEc2TestCase(test.TestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): super(ApiEc2TestCase, self).setUp() diff --git a/nova/tests/test_log.py b/nova/tests/test_log.py index beb1d97cf27f..868a5ead392e 100644 --- a/nova/tests/test_log.py +++ b/nova/tests/test_log.py @@ -9,7 +9,7 @@ def _fake_context(): return context.RequestContext(1, 1) -class RootLoggerTestCase(test.TrialTestCase): +class RootLoggerTestCase(test.TestCase): def setUp(self): super(RootLoggerTestCase, self).setUp() self.log = log.logging.root @@ -46,7 +46,7 @@ class RootLoggerTestCase(test.TrialTestCase): self.assert_(True) # didn't raise exception -class NovaFormatterTestCase(test.TrialTestCase): +class NovaFormatterTestCase(test.TestCase): def setUp(self): super(NovaFormatterTestCase, self).setUp() self.flags(logging_context_format_string="HAS CONTEXT "\ @@ -78,7 +78,7 @@ class NovaFormatterTestCase(test.TrialTestCase): self.assertEqual("NOCTXT: baz --DBG\n", self.stream.getvalue()) -class NovaLoggerTestCase(test.TrialTestCase): +class NovaLoggerTestCase(test.TestCase): def setUp(self): super(NovaLoggerTestCase, self).setUp() self.flags(default_log_levels=["nova-test=AUDIT"], verbose=False) @@ -96,7 +96,7 @@ class NovaLoggerTestCase(test.TrialTestCase): self.assertEqual(log.AUDIT, l.level) -class VerboseLoggerTestCase(test.TrialTestCase): +class VerboseLoggerTestCase(test.TestCase): def setUp(self): super(VerboseLoggerTestCase, self).setUp() self.flags(default_log_levels=["nova.test=AUDIT"], verbose=True) diff --git a/nova/tests/test_middleware.py b/nova/tests/test_middleware.py index 0febf52d6c0b..9d49167baf8b 100644 --- a/nova/tests/test_middleware.py +++ b/nova/tests/test_middleware.py @@ -38,7 +38,7 @@ def conditional_forbid(req): return 'OK' -class LockoutTestCase(test.TrialTestCase): +class LockoutTestCase(test.TestCase): """Test case for the Lockout middleware.""" def setUp(self): # pylint: disable-msg=C0103 super(LockoutTestCase, self).setUp() diff --git a/nova/tests/test_twistd.py b/nova/tests/test_twistd.py index 75007b9c87e9..ff8627c3b550 100644 --- a/nova/tests/test_twistd.py +++ b/nova/tests/test_twistd.py @@ -28,7 +28,7 @@ from nova import test FLAGS = flags.FLAGS -class TwistdTestCase(test.TrialTestCase): +class TwistdTestCase(test.TestCase): def setUp(self): super(TwistdTestCase, self).setUp() self.Options = twistd.WrapTwistedOptions(twistd.TwistdServerOptions) From f16030423d43272c5c3bea7fe51a0e03f0d95846 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 14 Jan 2011 11:19:51 -0800 Subject: [PATCH 79/84] undo accidental removal of fake_flags --- nova/test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/test.py b/nova/test.py index a26b85b04ab3..881baccd5c1f 100644 --- a/nova/test.py +++ b/nova/test.py @@ -27,8 +27,6 @@ import unittest import mox import stubout -from twisted.internet import defer -from twisted.trial import unittest as trial_unittest from nova import context from nova import db @@ -36,6 +34,7 @@ from nova import fakerabbit from nova import flags from nova import rpc from nova.network import manager as network_manager +from nova.tests import fake_flags FLAGS = flags.FLAGS From bf0d75e6f78bc3c66dce8481d44e52c40a9addb0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 14 Jan 2011 11:20:46 -0800 Subject: [PATCH 80/84] fix bad function signature in create_networks --- nova/network/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/network/manager.py b/nova/network/manager.py index 4d553f074b0c..a92bd3f997f1 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -239,8 +239,8 @@ class NetworkManager(manager.Manager): """Get the network host for the current context.""" raise NotImplementedError() - def create_networks(self, context, num_networks, network_size, cidr_v6, - *args, **kwargs): + def create_networks(self, context, cidr, num_networks, network_size, + cidr_v6, *args, **kwargs): """Create networks based on parameters.""" raise NotImplementedError() From d0713a6a2149274eeeef6fd22e7da4706a8190ec Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 14 Jan 2011 11:36:48 -0800 Subject: [PATCH 81/84] removed rpc in cloud --- nova/api/ec2/cloud.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 4adcb7415456..fb7e6a59f528 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -37,7 +37,6 @@ from nova import exception from nova import flags from nova import log as logging from nova import network -from nova import rpc from nova import utils from nova import volume from nova.compute import instance_types From 4f920a8316afc4becdabbc0a75959a8e8017836f Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Fri, 14 Jan 2011 17:43:34 -0800 Subject: [PATCH 82/84] remove print statement --- nova/tests/api/openstack/test_servers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index bb598ddeb5be..0396daf98634 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -101,7 +101,6 @@ class ServersTest(unittest.TestCase): def test_get_server_by_id(self): req = webob.Request.blank('/v1.0/servers/1') res = req.get_response(nova.api.API('os')) - print res.body res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') self.assertEqual(res_dict['server']['name'], 'server1') From 731126b299da757588656fa72b291ca4da96b5fe Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Fri, 14 Jan 2011 17:44:47 -0800 Subject: [PATCH 83/84] pep8 --- nova/db/sqlalchemy/models.py | 2 +- nova/tests/test_console.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 0e56ea7e69b9..e8fb1f439a9d 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -91,7 +91,7 @@ class NovaBase(object): def iteritems(self): """Make the model object behave like a dict. - + Includes attributes from joins.""" local = dict(self) joined = dict([(k, v) for k, v in self.__dict__.iteritems() diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 69e2109e6373..85bf94458223 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -130,4 +130,3 @@ class ConsoleTestCase(test.TestCase): self.context, console_id) db.instance_destroy(self.context, instance_id) - From 69c11c27c20c74aced491ecfe78a80872ad6232a Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Fri, 14 Jan 2011 17:54:36 -0800 Subject: [PATCH 84/84] pep8 fixes... largely to things from trunk? --- nova/api/ec2/cloud.py | 3 ++- nova/db/sqlalchemy/api.py | 6 +++--- nova/network/manager.py | 6 ++---- nova/virt/libvirt_conn.py | 3 +-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index d1aec23be052..a1ae70fb0c51 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -515,7 +515,8 @@ class CloudController(object): # instance_id is passed in as a list of instances ec2_id = instance_id[0] instance_id = ec2_id_to_id(ec2_id) - output = self.compute_api.get_console_output(context, instance_id=instance_id) + output = self.compute_api.get_console_output( + context, instance_id=instance_id) now = datetime.datetime.utcnow() return {"InstanceId": ec2_id, "Timestamp": now, diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 3ba69af9e454..b63b84bed595 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -611,9 +611,9 @@ def fixed_ip_get_instance_v6(context, address): session = get_session() mac = utils.to_mac(address) - result = session.query(models.Instance - ).filter_by(mac_address=mac - ).first() + result = session.query(models.Instance).\ + filter_by(mac_address=mac).\ + first() return result diff --git a/nova/network/manager.py b/nova/network/manager.py index a92bd3f997f1..2a043cc6bf30 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -517,10 +517,8 @@ class VlanManager(NetworkManager): net['vlan'] = vlan net['bridge'] = 'br%s' % vlan if(FLAGS.use_ipv6): - cidr_v6 = "%s/%s" % ( - fixed_net_v6[start_v6], - significant_bits_v6 - ) + cidr_v6 = "%s/%s" % (fixed_net_v6[start_v6], + significant_bits_v6) net['cidr_v6'] = cidr_v6 # NOTE(vish): This makes ports unique accross the cloud, a more diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 073a8e5bb3a2..b06246135c72 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -995,8 +995,7 @@ class NWFilterFirewall(FirewallDriver): ['no-mac-spoofing', 'no-ip-spoofing', 'no-arp-spoofing', - 'allow-dhcp-server' - ])) + 'allow-dhcp-server'])) self._define_filter(self.nova_base_ipv4_filter) self._define_filter(self.nova_base_ipv6_filter) self._define_filter(self.nova_dhcp_filter)