From 0e13442d2cd83b62361b9c611d987bdcb099cf25 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 26 Jul 2010 23:19:51 -0400 Subject: [PATCH 01/64] Share my updates to the Rackspace API. --- bin/nova-rsapi | 13 +--- exercise_rsapi.py | 51 ------------- nova/endpoint/rackspace.py | 147 +++++++++++++++++++++++++++---------- 3 files changed, 109 insertions(+), 102 deletions(-) delete mode 100644 exercise_rsapi.py diff --git a/bin/nova-rsapi b/bin/nova-rsapi index c2f2c9d7..a529fc66 100755 --- a/bin/nova-rsapi +++ b/bin/nova-rsapi @@ -37,23 +37,12 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') def main(_argv): - user_manager = users.UserManager() - api_instance = rackspace.Api(user_manager) - conn = rpc.Connection.instance() - rpc_consumer = rpc.AdapterConsumer(connection=conn, - topic=FLAGS.cloud_topic, - proxy=api_instance) - -# TODO: fire rpc response listener (without attach to tornado) -# io_inst = ioloop.IOLoop.instance() -# _injected = consumer.attach_to_tornado(io_inst) - + api_instance = rackspace.Api() http_server = simple_server.WSGIServer(('0.0.0.0', FLAGS.cc_port), simple_server.WSGIRequestHandler) http_server.set_app(api_instance.handler) logging.debug('Started HTTP server on port %i' % FLAGS.cc_port) while True: http_server.handle_request() -# io_inst.start() if __name__ == '__main__': utils.default_flagfile() diff --git a/exercise_rsapi.py b/exercise_rsapi.py deleted file mode 100644 index 20589b9c..00000000 --- a/exercise_rsapi.py +++ /dev/null @@ -1,51 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import cloudservers - -class IdFake: - def __init__(self, id): - self.id = id - -# to get your access key: -# from nova.auth import users -# users.UserManger.instance().get_users()[0].access -rscloud = cloudservers.CloudServers( - 'admin', - '6cca875e-5ab3-4c60-9852-abf5c5c60cc6' - ) -rscloud.client.AUTH_URL = 'http://localhost:8773/v1.0' - - -rv = rscloud.servers.list() -print "SERVERS: %s" % rv - -if len(rv) == 0: - server = rscloud.servers.create( - "test-server", - IdFake("ami-tiny"), - IdFake("m1.tiny") - ) - print "LAUNCH: %s" % server -else: - server = rv[0] - print "Server to kill: %s" % server - -raw_input("press enter key to kill the server") - -server.delete() diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 29a077b2..b561212f 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -48,82 +48,128 @@ FLAGS = flags.FLAGS flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') -# TODO(todd): subclass Exception so we can bubble meaningful errors +class Unauthorized(Exception): + pass + +class NotFound(Exception): + pass class Api(object): - def __init__(self, rpc_mechanism): + def __init__(self): + """build endpoints here""" self.controllers = { "v1.0": RackspaceAuthenticationApi(), "servers": RackspaceCloudServerApi() } - self.rpc_mechanism = rpc_mechanism def handler(self, environ, responder): + """ + This is the entrypoint from wsgi. Read PEP 333 and wsgi.org for + more intormation. The key points are responder is a callback that + needs to run before you return, and takes two arguments, response + code string ("200 OK") and headers (["X-How-Cool-Am-I: Ultra-Suede"]) + and the return value is the body of the response. + """ environ['nova.context'] = self.build_context(environ) controller, path = wsgi.Util.route( environ['PATH_INFO'], self.controllers ) + logging.debug("Route %s to %s", str(path), str(controller)) if not controller: - # TODO(todd): Exception (404) - raise Exception("Missing Controller") - rv = controller.process(path, environ) - if type(rv) is tuple: - responder(rv[0], rv[1]) - rv = rv[2] - else: - responder("200 OK", []) - return rv + responder("404 Not Found", []) + return "" + try: + rv = controller.process(path, environ) + if type(rv) is tuple: + responder(rv[0], rv[1]) + rv = rv[2] + else: + responder("200 OK", []) + return rv + except Unauthorized: + responder("401 Unauthorized", []) + return "" + except NotFound: + responder("404 Not Found", []) + return "" + def build_context(self, env): rv = {} if env.has_key("HTTP_X_AUTH_TOKEN"): + # TODO(todd): once we make an actual unique token, this will change rv['user'] = users.UserManager.instance().get_user_from_access_key( - env['HTTP_X_AUTH_TOKEN'] - ) + env['HTTP_X_AUTH_TOKEN']) if rv['user']: rv['project'] = users.UserManager.instance().get_project( - rv['user'].name - ) + rv['user'].name) return rv class RackspaceApiEndpoint(object): def process(self, path, env): + """ + Main entrypoint for all controllers (what gets run by the wsgi handler). + Check authentication based on key, raise Unauthorized if invalid. + + Select the most appropriate action based on request type GET, POST, etc, + then pass it through to the implementing controller. Defalut to GET if + the implementing child doesn't respond to a particular type. + """ if not self.check_authentication(env): - # TODO(todd): Exception (Unauthorized) - raise Exception("Unable to authenticate") + raise Unauthorized("Unable to authenticate") - if len(path) == 0: + method = env['REQUEST_METHOD'].lower() + callback = getattr(self, method, None) + if not callback: + callback = getattr(self, "get") + logging.debug("%s processing %s with %s", self, method, callback) + return callback(path, env) + + def get(self, path, env): + """ + The default GET will look at the path and call an appropriate + action within this controller based on the the structure of the path. + + Given the following path lengths (with the first part stripped of by + router, as it is the controller name): + = 0 -> index + = 1 -> first component (/servers/details -> details) + >= 2 -> second path component (/servers/ID/ips/* -> ips) + + This should return + A String if 200 OK and no additional headers + (CODE, HEADERS, BODY) for custom response code and headers + """ + if len(path) == 0 and hasattr(self, "index"): + logging.debug("%s running index", self) return self.index(env) + if len(path) >= 2: + action = path[1] + else: + action = path.pop(0) - action = path.pop(0) + logging.debug("%s running action %s", self, action) if hasattr(self, action): method = getattr(self, action) return method(path, env) else: - # TODO(todd): Exception (404) - raise Exception("Missing method %s" % path[0]) + raise NotFound("Missing method %s" % path[0]) def check_authentication(self, env): - if hasattr(self, "process_without_authentication") \ - and getattr(self, "process_without_authentication"): - return True if not env['nova.context']['user']: return False return True -class RackspaceAuthenticationApi(RackspaceApiEndpoint): - - def __init__(self): - self.process_without_authentication = True +class RackspaceAuthenticationApi(object): # TODO(todd): make a actual session with a unique token # just pass the auth key back through for now - def index(self, env): + def index(self, _path, env): response = '204 No Content' headers = [ ('X-Server-Management-Url', 'http://%s' % env['HTTP_HOST']), @@ -141,20 +187,25 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): self.instdir = model.InstanceDirectory() self.network = network.PublicNetworkController() + def post(self, path, env): + if len(path) == 0: + return self.launch_server(env) + + def delete(self, path_parts, env): + if self.delete_server(path_parts[0]): + return ("202 Accepted", [], "") + else: + return ("404 Not Found", [], + "Did not find image, or it was not in a running state") + + def index(self, env): - if env['REQUEST_METHOD'] == 'GET': - return self.detail(env) - elif env['REQUEST_METHOD'] == 'POST': - return self.launch_server(env) + return self.detail(env) def detail(self, args, env): - value = { - "servers": - [] - } + value = {"servers": []} for inst in self.instdir.all: value["servers"].append(self.instance_details(inst)) - return json.dumps(value) ## @@ -227,3 +278,21 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): "args": {"instance_id": inst.instance_id} } ) + + def delete_server(self, instance_id): + owner_hostname = self.host_for_instance(instance_id) + # it isn't launched? + if not owner_hostname: + return None + rpc_transport = "%s:%s" % (FLAGS.compute_topic, owner_hostname) + rpc.cast(rpc_transport, + {"method": "reboot_instance", + "args": {"instance_id": instance_id}}) + return True + + def host_for_instance(self, instance_id): + instance = model.Instance.lookup(instance_id) + if not instance: + return None + return instance["node_name"] + From e57ac9a86492a654e4b495877ce1b4eb26860d7e Mon Sep 17 00:00:00 2001 From: "Joel Moore joelbm24@gmail.com" <> Date: Tue, 27 Jul 2010 15:35:20 -0700 Subject: [PATCH 02/64] fixed path to keys directory --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 8c6c0556..9d81d3bb 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -86,7 +86,7 @@ class CloudController(object): """ Ensure the keychains and folders exist. """ # Create keys folder, if it doesn't exist if not os.path.exists(FLAGS.keys_path): - os.makedirs(os.path.abspath(FLAGS.keys_path)) + os.makedirs(FLAGS.keys_path) # Gen root CA, if we don't have one root_ca_path = os.path.join(FLAGS.ca_path, FLAGS.ca_file) if not os.path.exists(root_ca_path): From e70e2a10cc5deb196d834990f21a1a796c2705e7 Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 29 Jul 2010 19:53:00 +0200 Subject: [PATCH 03/64] Add some useful features to our flags * No longer dies if there are unknown flags. * Allows you to declare that you will use a flag from another file * Allows you to import new flags at runtime and reparses the original arguments to fill them once they are accessed. --- nova/flags.py | 133 +++++++++++++++++++++++++++++++++-- nova/tests/declare_flags.py | 5 ++ nova/tests/flags_unittest.py | 94 +++++++++++++++++++++++++ nova/tests/runtime_flags.py | 5 ++ run_tests.py | 1 + 5 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 nova/tests/declare_flags.py create mode 100644 nova/tests/flags_unittest.py create mode 100644 nova/tests/runtime_flags.py diff --git a/nova/flags.py b/nova/flags.py index f35f5fa1..2ec7d9c9 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -21,16 +21,137 @@ Package-level global flags are defined here, the rest are defined where they're used. """ +import getopt import socket +import sys + +import gflags -from gflags import * +class FlagValues(gflags.FlagValues): + def __init__(self): + gflags.FlagValues.__init__(self) + self.__dict__['__dirty'] = [] + self.__dict__['__was_already_parsed'] = False + self.__dict__['__stored_argv'] = [] + + def __call__(self, argv): + # We're doing some hacky stuff here so that we don't have to copy + # out all the code of the original verbatim and then tweak a few lines. + # We're hijacking the output of getopt so we can still return the + # leftover args at the end + sneaky_unparsed_args = {"value": None} + original_argv = list(argv) + + if self.IsGnuGetOpt(): + orig_getopt = getattr(getopt, 'gnu_getopt') + orig_name = 'gnu_getopt' + else: + orig_getopt = getattr(getopt, 'getopt') + orig_name = 'getopt' + + def _sneaky(*args, **kw): + optlist, unparsed_args = orig_getopt(*args, **kw) + sneaky_unparsed_args['value'] = unparsed_args + return optlist, unparsed_args + + try: + setattr(getopt, orig_name, _sneaky) + args = gflags.FlagValues.__call__(self, argv) + except gflags.UnrecognizedFlagError: + # Undefined args were found, for now we don't care so just + # act like everything went well + # (these three lines are copied pretty much verbatim from the end + # of the __call__ function we are wrapping) + unparsed_args = sneaky_unparsed_args['value'] + if unparsed_args: + if self.IsGnuGetOpt(): + args = argv[:1] + unparsed + else: + args = argv[:1] + original_argv[-len(unparsed_args):] + else: + args = argv[:1] + finally: + setattr(getopt, orig_name, orig_getopt) + + # Store the arguments for later, we'll need them for new flags + # added at runtime + self.__dict__['__stored_argv'] = original_argv + self.__dict__['__was_already_parsed'] = True + self.ClearDirty() + return args + + def SetDirty(self, name): + """Mark a flag as dirty so that accessing it will case a reparse.""" + self.__dict__['__dirty'].append(name) + + def IsDirty(self, name): + return name in self.__dict__['__dirty'] + + def ClearDirty(self): + self.__dict__['__is_dirty'] = [] + + def WasAlreadyParsed(self): + return self.__dict__['__was_already_parsed'] + + def ParseNewFlags(self): + if '__stored_argv' not in self.__dict__: + return + new_flags = FlagValues() + for k in self.__dict__['__dirty']: + new_flags[k] = gflags.FlagValues.__getitem__(self, k) + + new_flags(self.__dict__['__stored_argv']) + for k in self.__dict__['__dirty']: + setattr(self, k, getattr(new_flags, k)) + self.ClearDirty() + + def __setitem__(self, name, flag): + gflags.FlagValues.__setitem__(self, name, flag) + if self.WasAlreadyParsed(): + self.SetDirty(name) + + def __getitem__(self, name): + if self.IsDirty(name): + self.ParseNewFlags() + return gflags.FlagValues.__getitem__(self, name) + + def __getattr__(self, name): + if self.IsDirty(name): + self.ParseNewFlags() + return gflags.FlagValues.__getattr__(self, name) + + +FLAGS = FlagValues() + + +def party_wrapper(func): + def _wrapped(*args, **kw): + kw.setdefault('flag_values', FLAGS) + func(*args, **kw) + _wrapped.func_name = func.func_name + return _wrapped + + +DEFINE_string = party_wrapper(gflags.DEFINE_string) +DEFINE_integer = party_wrapper(gflags.DEFINE_integer) +DEFINE_bool = party_wrapper(gflags.DEFINE_bool) +DEFINE_boolean = party_wrapper(gflags.DEFINE_boolean) +DEFINE_float = party_wrapper(gflags.DEFINE_float) +DEFINE_enum = party_wrapper(gflags.DEFINE_enum) +DEFINE_list = party_wrapper(gflags.DEFINE_list) +DEFINE_spaceseplist = party_wrapper(gflags.DEFINE_spaceseplist) +DEFINE_multistring = party_wrapper(gflags.DEFINE_multistring) +DEFINE_multi_int = party_wrapper(gflags.DEFINE_multi_int) + + +def DECLARE(name, module_string, flag_values=FLAGS): + if module_string not in sys.modules: + __import__(module_string, globals(), locals()) + if name not in flag_values: + raise gflags.UnrecognizedFlag( + "%s not defined by %s" % (name, module_string)) -# This keeps pylint from barfing on the imports -FLAGS = FLAGS -DEFINE_string = DEFINE_string -DEFINE_integer = DEFINE_integer -DEFINE_bool = DEFINE_bool # __GLOBAL FLAGS ONLY__ # Define any app-specific flags in their own files, docs at: diff --git a/nova/tests/declare_flags.py b/nova/tests/declare_flags.py new file mode 100644 index 00000000..f7c91f9d --- /dev/null +++ b/nova/tests/declare_flags.py @@ -0,0 +1,5 @@ +from nova import flags + +FLAGS = flags.FLAGS + +flags.DEFINE_integer('answer', 42, 'test flag') diff --git a/nova/tests/flags_unittest.py b/nova/tests/flags_unittest.py new file mode 100644 index 00000000..44da94d0 --- /dev/null +++ b/nova/tests/flags_unittest.py @@ -0,0 +1,94 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +from twisted.internet import defer +from twisted.internet import reactor +from xml.etree import ElementTree + +from nova import exception +from nova import flags +from nova import process +from nova import test +from nova import utils + + +class FlagsTestCase(test.TrialTestCase): + def setUp(self): + super(FlagsTestCase, self).setUp() + self.FLAGS = flags.FlagValues() + self.global_FLAGS = flags.FLAGS + + def test_define(self): + self.assert_('string' not in self.FLAGS) + self.assert_('int' not in self.FLAGS) + self.assert_('false' not in self.FLAGS) + self.assert_('true' not in self.FLAGS) + + flags.DEFINE_string('string', 'default', 'desc', flag_values=self.FLAGS) + flags.DEFINE_integer('int', 1, 'desc', flag_values=self.FLAGS) + flags.DEFINE_bool('false', False, 'desc', flag_values=self.FLAGS) + flags.DEFINE_bool('true', True, 'desc', flag_values=self.FLAGS) + + self.assert_(self.FLAGS['string']) + self.assert_(self.FLAGS['int']) + self.assert_(self.FLAGS['false']) + self.assert_(self.FLAGS['true']) + self.assertEqual(self.FLAGS.string, 'default') + self.assertEqual(self.FLAGS.int, 1) + self.assertEqual(self.FLAGS.false, False) + self.assertEqual(self.FLAGS.true, True) + + argv = ['flags_test', + '--string', 'foo', + '--int', '2', + '--false', + '--notrue'] + + self.FLAGS(argv) + self.assertEqual(self.FLAGS.string, 'foo') + self.assertEqual(self.FLAGS.int, 2) + self.assertEqual(self.FLAGS.false, True) + self.assertEqual(self.FLAGS.true, False) + + def test_declare(self): + self.assert_('answer' not in self.global_FLAGS) + flags.DECLARE('answer', 'nova.tests.declare_flags') + self.assert_('answer' in self.global_FLAGS) + self.assertEqual(self.global_FLAGS.answer, 42) + + # Make sure we don't overwrite anything + self.global_FLAGS.answer = 256 + self.assertEqual(self.global_FLAGS.answer, 256) + flags.DECLARE('answer', 'nova.tests.declare_flags') + self.assertEqual(self.global_FLAGS.answer, 256) + + def test_runtime_and_unknown_flags(self): + self.assert_('runtime_answer' not in self.global_FLAGS) + + argv = ['flags_test', '--runtime_answer=60', 'extra_arg'] + args = self.global_FLAGS(argv) + self.assertEqual(len(args), 2) + self.assertEqual(args[1], 'extra_arg') + + self.assert_('runtime_answer' not in self.global_FLAGS) + + import nova.tests.runtime_flags + + self.assert_('runtime_answer' in self.global_FLAGS) + self.assertEqual(self.global_FLAGS.runtime_answer, 60) diff --git a/nova/tests/runtime_flags.py b/nova/tests/runtime_flags.py new file mode 100644 index 00000000..a2cc4738 --- /dev/null +++ b/nova/tests/runtime_flags.py @@ -0,0 +1,5 @@ +from nova import flags + +FLAGS = flags.FLAGS + +flags.DEFINE_integer('runtime_answer', 54, 'test flag') diff --git a/run_tests.py b/run_tests.py index 5a8966f0..14019b65 100644 --- a/run_tests.py +++ b/run_tests.py @@ -54,6 +54,7 @@ from nova.tests.auth_unittest import * from nova.tests.api_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * +from nova.tests.flags_unittest import * from nova.tests.model_unittest import * from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * From 94fbe244d6512a11951e0c38e258d65f2b351c5d Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 29 Jul 2010 20:05:22 +0200 Subject: [PATCH 04/64] strip out some useless imports --- nova/tests/flags_unittest.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/nova/tests/flags_unittest.py b/nova/tests/flags_unittest.py index 44da94d0..d49d5dc4 100644 --- a/nova/tests/flags_unittest.py +++ b/nova/tests/flags_unittest.py @@ -16,16 +16,9 @@ # License for the specific language governing permissions and limitations # under the License. -import logging -from twisted.internet import defer -from twisted.internet import reactor -from xml.etree import ElementTree - from nova import exception from nova import flags -from nova import process from nova import test -from nova import utils class FlagsTestCase(test.TrialTestCase): From e3b9d0bffb825013b2b8c783dbdb17f0c0f753c3 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Tue, 3 Aug 2010 00:52:06 +0100 Subject: [PATCH 05/64] Added a xapi plugin that can pull images from nova-objectstore, and use that to get a disk, kernel, and ramdisk for the VM. The VM actually boots! --- xenapi/README | 2 + xenapi/etc/xapi.d/plugins/objectstore | 231 ++++++++++++++++++++ xenapi/etc/xapi.d/plugins/pluginlib_nova.py | 216 ++++++++++++++++++ 3 files changed, 449 insertions(+) create mode 100644 xenapi/README create mode 100644 xenapi/etc/xapi.d/plugins/objectstore create mode 100755 xenapi/etc/xapi.d/plugins/pluginlib_nova.py diff --git a/xenapi/README b/xenapi/README new file mode 100644 index 00000000..1fc67aa7 --- /dev/null +++ b/xenapi/README @@ -0,0 +1,2 @@ +This directory contains files that are required for the XenAPI support. They +should be installed in the XenServer / Xen Cloud Platform domain 0. diff --git a/xenapi/etc/xapi.d/plugins/objectstore b/xenapi/etc/xapi.d/plugins/objectstore new file mode 100644 index 00000000..271e7337 --- /dev/null +++ b/xenapi/etc/xapi.d/plugins/objectstore @@ -0,0 +1,231 @@ +#!/usr/bin/env python + +# Copyright (c) 2010 Citrix Systems, Inc. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# +# XenAPI plugin for fetching images from nova-objectstore. +# + +import base64 +import errno +import hmac +import os +import os.path +import sha +import time +import urlparse + +import XenAPIPlugin + +from pluginlib_nova import * +configure_logging('objectstore') + + +KERNEL_DIR = '/boot/guest' + +DOWNLOAD_CHUNK_SIZE = 2 * 1024 * 1024 +SECTOR_SIZE = 512 +MBR_SIZE_SECTORS = 63 +MBR_SIZE_BYTES = MBR_SIZE_SECTORS * SECTOR_SIZE + + +def get_vdi(session, args): + src_url = exists(args, 'src_url') + username = exists(args, 'username') + password = exists(args, 'password') + add_partition = validate_bool(args, 'add_partition', 'false') + + (proto, netloc, url_path, _, _, _) = urlparse.urlparse(src_url) + + sr = find_sr(session) + if sr is None: + raise Exception('Cannot find SR to write VDI to') + + virtual_size = \ + get_content_length(proto, netloc, url_path, username, password) + if virtual_size < 0: + raise Exception('Cannot get VDI size') + + vdi_size = virtual_size + if add_partition: + # Make room for MBR. + vdi_size += MBR_SIZE_BYTES + + vdi = create_vdi(session, sr, src_url, vdi_size, False) + with_vdi_in_dom0(session, vdi, False, + lambda dev: get_vdi_(proto, netloc, url_path, + username, password, add_partition, + virtual_size, '/dev/%s' % dev)) + return session.xenapi.VDI.get_uuid(vdi) + + +def get_vdi_(proto, netloc, url_path, username, password, add_partition, + virtual_size, dest): + + if add_partition: + write_partition(virtual_size, dest) + + offset = add_partition and MBR_SIZE_BYTES or 0 + get(proto, netloc, url_path, username, password, dest, offset) + + +def write_partition(virtual_size, dest): + mbr_last = MBR_SIZE_SECTORS - 1 + primary_first = MBR_SIZE_SECTORS + primary_last = MBR_SIZE_SECTORS + (virtual_size / SECTOR_SIZE) - 1 + + logging.debug('Writing partition table %d %d to %s...', + primary_first, primary_last, dest) + + result = os.system('parted --script %s mklabel msdos' % dest) + if result != 0: + raise Exception('Failed to mklabel') + result = os.system('parted --script %s mkpart primary %ds %ds' % + (dest, primary_first, primary_last)) + if result != 0: + raise Exception('Failed to mkpart') + + logging.debug('Writing partition table %s done.', dest) + + +def find_sr(session): + host = get_this_host(session) + srs = session.xenapi.SR.get_all() + for sr in srs: + sr_rec = session.xenapi.SR.get_record(sr) + if not ('i18n-key' in sr_rec['other_config'] and + sr_rec['other_config']['i18n-key'] == 'local-storage'): + continue + for pbd in sr_rec['PBDs']: + pbd_rec = session.xenapi.PBD.get_record(pbd) + if pbd_rec['host'] == host: + return sr + return None + + +def get_kernel(session, args): + src_url = exists(args, 'src_url') + username = exists(args, 'username') + password = exists(args, 'password') + + (proto, netloc, url_path, _, _, _) = urlparse.urlparse(src_url) + + dest = os.path.join(KERNEL_DIR, url_path[1:]) + + # Paranoid check against people using ../ to do rude things. + if os.path.commonprefix([KERNEL_DIR, dest]) != KERNEL_DIR: + raise Exception('Illegal destination %s %s', (url_path, dest)) + + dirname = os.path.dirname(dest) + try: + os.makedirs(dirname) + except os.error, e: + if e.errno != errno.EEXIST: + raise + if not os.path.isdir(dirname): + raise Exception('Cannot make directory %s', dirname) + + try: + os.remove(dest) + except: + pass + + get(proto, netloc, url_path, username, password, dest, 0) + + return dest + + +def get_content_length(proto, netloc, url_path, username, password): + headers = make_headers('HEAD', url_path, username, password) + return with_http_connection( + proto, netloc, + lambda conn: get_content_length_(url_path, headers, conn)) + + +def get_content_length_(url_path, headers, conn): + conn.request('HEAD', url_path, None, headers) + response = conn.getresponse() + if response.status != 200: + raise Exception('%d %s' % (response.status, response.reason)) + + return long(response.getheader('Content-Length', -1)) + + +def get(proto, netloc, url_path, username, password, dest, offset): + headers = make_headers('GET', url_path, username, password) + download(proto, netloc, url_path, headers, dest, offset) + + +def make_headers(verb, url_path, username, password): + headers = {} + headers['Date'] = \ + time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) + headers['Authorization'] = \ + 'AWS %s:%s' % (username, + s3_authorization(verb, url_path, password, headers)) + return headers + + +def s3_authorization(verb, path, password, headers): + sha1 = hmac.new(password, digestmod=sha) + sha1.update(plaintext(verb, path, headers)) + return base64.encodestring(sha1.digest()).strip() + + +def plaintext(verb, path, headers): + return '%s\n\n\n%s\n%s' % (verb, + "\n".join([headers[h] for h in headers]), + path) + + +def download(proto, netloc, url_path, headers, dest, offset): + with_http_connection( + proto, netloc, + lambda conn: download_(url_path, dest, offset, headers, conn)) + + +def download_(url_path, dest, offset, headers, conn): + conn.request('GET', url_path, None, headers) + response = conn.getresponse() + if response.status != 200: + raise Exception('%d %s' % (response.status, response.reason)) + + length = response.getheader('Content-Length', -1) + + with_file( + dest, 'a', + lambda dest_file: download_all(response, length, dest_file, offset)) + + +def download_all(response, length, dest_file, offset): + dest_file.seek(offset) + i = 0 + while True: + buf = response.read(DOWNLOAD_CHUNK_SIZE) + if buf: + dest_file.write(buf) + else: + return + i += len(buf) + if length != -1 and i >= length: + return + + +if __name__ == '__main__': + XenAPIPlugin.dispatch({'get_vdi': get_vdi, + 'get_kernel': get_kernel}) diff --git a/xenapi/etc/xapi.d/plugins/pluginlib_nova.py b/xenapi/etc/xapi.d/plugins/pluginlib_nova.py new file mode 100755 index 00000000..2d323a01 --- /dev/null +++ b/xenapi/etc/xapi.d/plugins/pluginlib_nova.py @@ -0,0 +1,216 @@ +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# +# Helper functions for the Nova xapi plugins. In time, this will merge +# with the pluginlib.py shipped with xapi, but for now, that file is not +# very stable, so it's easiest just to have a copy of all the functions +# that we need. +# + +import httplib +import logging +import logging.handlers +import re +import time + + +##### Logging setup + +def configure_logging(name): + log = logging.getLogger() + log.setLevel(logging.DEBUG) + sysh = logging.handlers.SysLogHandler('/dev/log') + sysh.setLevel(logging.DEBUG) + formatter = logging.Formatter('%s: %%(levelname)-8s %%(message)s' % name) + sysh.setFormatter(formatter) + log.addHandler(sysh) + + +##### Exceptions + +class PluginError(Exception): + """Base Exception class for all plugin errors.""" + def __init__(self, *args): + Exception.__init__(self, *args) + +class ArgumentError(PluginError): + """Raised when required arguments are missing, argument values are invalid, + or incompatible arguments are given. + """ + def __init__(self, *args): + PluginError.__init__(self, *args) + + +##### Helpers + +def ignore_failure(func, *args, **kwargs): + try: + return func(*args, **kwargs) + except XenAPI.Failure, e: + logging.error('Ignoring XenAPI.Failure %s', e) + return None + + +##### Argument validation + +ARGUMENT_PATTERN = re.compile(r'^[a-zA-Z0-9_:\.\-,]+$') + +def validate_exists(args, key, default=None): + """Validates that a string argument to a RPC method call is given, and + matches the shell-safe regex, with an optional default value in case it + does not exist. + + Returns the string. + """ + if key in args: + if len(args[key]) == 0: + raise ArgumentError('Argument %r value %r is too short.' % (key, args[key])) + if not ARGUMENT_PATTERN.match(args[key]): + raise ArgumentError('Argument %r value %r contains invalid characters.' % (key, args[key])) + if args[key][0] == '-': + raise ArgumentError('Argument %r value %r starts with a hyphen.' % (key, args[key])) + return args[key] + elif default is not None: + return default + else: + raise ArgumentError('Argument %s is required.' % key) + +def validate_bool(args, key, default=None): + """Validates that a string argument to a RPC method call is a boolean string, + with an optional default value in case it does not exist. + + Returns the python boolean value. + """ + value = validate_exists(args, key, default) + if value.lower() == 'true': + return True + elif value.lower() == 'false': + return False + else: + raise ArgumentError("Argument %s may not take value %r. Valid values are ['true', 'false']." % (key, value)) + +def exists(args, key): + """Validates that a freeform string argument to a RPC method call is given. + Returns the string. + """ + if key in args: + return args[key] + else: + raise ArgumentError('Argument %s is required.' % key) + +def optional(args, key): + """If the given key is in args, return the corresponding value, otherwise + return None""" + return key in args and args[key] or None + + +def get_this_host(session): + return session.xenapi.session.get_this_host(session.handle) + + +def get_domain_0(session): + this_host_ref = get_this_host(session) + expr = 'field "is_control_domain" = "true" and field "resident_on" = "%s"' % this_host_ref + return session.xenapi.VM.get_all_records_where(expr).keys()[0] + + +def create_vdi(session, sr_ref, name_label, virtual_size, read_only): + vdi_ref = session.xenapi.VDI.create( + { 'name_label': name_label, + 'name_description': '', + 'SR': sr_ref, + 'virtual_size': str(virtual_size), + 'type': 'User', + 'sharable': False, + 'read_only': read_only, + 'xenstore_data': {}, + 'other_config': {}, + 'sm_config': {}, + 'tags': [] }) + logging.debug('Created VDI %s (%s, %s, %s) on %s.', vdi_ref, name_label, + virtual_size, read_only, sr_ref) + return vdi_ref + + +def with_vdi_in_dom0(session, vdi, read_only, f): + dom0 = get_domain_0(session) + vbd_rec = {} + vbd_rec['VM'] = dom0 + vbd_rec['VDI'] = vdi + vbd_rec['userdevice'] = 'autodetect' + vbd_rec['bootable'] = False + vbd_rec['mode'] = read_only and 'RO' or 'RW' + vbd_rec['type'] = 'disk' + vbd_rec['unpluggable'] = True + vbd_rec['empty'] = False + vbd_rec['other_config'] = {} + vbd_rec['qos_algorithm_type'] = '' + vbd_rec['qos_algorithm_params'] = {} + vbd_rec['qos_supported_algorithms'] = [] + logging.debug('Creating VBD for VDI %s ... ', vdi) + vbd = session.xenapi.VBD.create(vbd_rec) + logging.debug('Creating VBD for VDI %s done.', vdi) + try: + logging.debug('Plugging VBD %s ... ', vbd) + session.xenapi.VBD.plug(vbd) + logging.debug('Plugging VBD %s done.', vbd) + return f(session.xenapi.VBD.get_device(vbd)) + finally: + logging.debug('Destroying VBD for VDI %s ... ', vdi) + vbd_unplug_with_retry(session, vbd) + ignore_failure(session.xenapi.VBD.destroy, vbd) + logging.debug('Destroying VBD for VDI %s done.', vdi) + + +def vbd_unplug_with_retry(session, vbd): + """Call VBD.unplug on the given VBD, with a retry if we get + DEVICE_DETACH_REJECTED. For reasons which I don't understand, we're + seeing the device still in use, even when all processes using the device + should be dead.""" + while True: + try: + session.xenapi.VBD.unplug(vbd) + logging.debug('VBD.unplug successful first time.') + return + except XenAPI.Failure, e: + if (len(e.details) > 0 and + e.details[0] == 'DEVICE_DETACH_REJECTED'): + logging.debug('VBD.unplug rejected: retrying...') + time.sleep(1) + elif (len(e.details) > 0 and + e.details[0] == 'DEVICE_ALREADY_DETACHED'): + logging.debug('VBD.unplug successful eventually.') + return + else: + logging.error('Ignoring XenAPI.Failure in VBD.unplug: %s', e) + return + + +def with_http_connection(proto, netloc, f): + conn = (proto == 'https' and + httplib.HTTPSConnection(netloc) or + httplib.HTTPConnection(netloc)) + try: + return f(conn) + finally: + conn.close() + + +def with_file(dest_path, mode, f): + dest = open(dest_path, mode) + try: + return f(dest) + finally: + dest.close() From bbb07172177d9a2b6553cf8e0b8e89f9d9b9309a Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Tue, 3 Aug 2010 15:37:31 +0100 Subject: [PATCH 06/64] Define __contains__ on BasicModel, so that we can use "x in datamodel". --- nova/datastore.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/datastore.py b/nova/datastore.py index 9c259233..f6c11d2c 100644 --- a/nova/datastore.py +++ b/nova/datastore.py @@ -168,6 +168,9 @@ class BasicModel(object): def setdefault(self, item, default): return self.state.setdefault(item, default) + def __contains__(self, item): + return item in self.state + def __getitem__(self, item): return self.state[item] From 0e216d0fe0e836770a7ee7558d3c91c5fe13fdc1 Mon Sep 17 00:00:00 2001 From: andy Date: Tue, 3 Aug 2010 16:51:37 +0200 Subject: [PATCH 07/64] add copyright headers --- nova/tests/declare_flags.py | 18 ++++++++++++++++++ nova/tests/runtime_flags.py | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/nova/tests/declare_flags.py b/nova/tests/declare_flags.py index f7c91f9d..51a55ec7 100644 --- a/nova/tests/declare_flags.py +++ b/nova/tests/declare_flags.py @@ -1,3 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + from nova import flags FLAGS = flags.FLAGS diff --git a/nova/tests/runtime_flags.py b/nova/tests/runtime_flags.py index a2cc4738..1eb50140 100644 --- a/nova/tests/runtime_flags.py +++ b/nova/tests/runtime_flags.py @@ -1,3 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + from nova import flags FLAGS = flags.FLAGS From f3568872af65898f3cb2f066205f6bcb6db638a2 Mon Sep 17 00:00:00 2001 From: andy Date: Tue, 3 Aug 2010 18:00:11 +0200 Subject: [PATCH 08/64] updated doc string and wrapper --- nova/flags.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 2ec7d9c9..b3bdd088 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -29,6 +29,14 @@ import gflags class FlagValues(gflags.FlagValues): + """Extension of gflags.FlagValues that allows undefined and runtime flags. + + Unknown flags will be ignored when parsing the command line, but the + command line will be kept so that it can be replayed if new flags are + defined after the initial parsing. + + """ + def __init__(self): gflags.FlagValues.__init__(self) self.__dict__['__dirty'] = [] @@ -125,7 +133,7 @@ class FlagValues(gflags.FlagValues): FLAGS = FlagValues() -def party_wrapper(func): +def _wrapper(func): def _wrapped(*args, **kw): kw.setdefault('flag_values', FLAGS) func(*args, **kw) @@ -133,16 +141,16 @@ def party_wrapper(func): return _wrapped -DEFINE_string = party_wrapper(gflags.DEFINE_string) -DEFINE_integer = party_wrapper(gflags.DEFINE_integer) -DEFINE_bool = party_wrapper(gflags.DEFINE_bool) -DEFINE_boolean = party_wrapper(gflags.DEFINE_boolean) -DEFINE_float = party_wrapper(gflags.DEFINE_float) -DEFINE_enum = party_wrapper(gflags.DEFINE_enum) -DEFINE_list = party_wrapper(gflags.DEFINE_list) -DEFINE_spaceseplist = party_wrapper(gflags.DEFINE_spaceseplist) -DEFINE_multistring = party_wrapper(gflags.DEFINE_multistring) -DEFINE_multi_int = party_wrapper(gflags.DEFINE_multi_int) +DEFINE_string = _wrapper(gflags.DEFINE_string) +DEFINE_integer = _wrapper(gflags.DEFINE_integer) +DEFINE_bool = _wrapper(gflags.DEFINE_bool) +DEFINE_boolean = _wrapper(gflags.DEFINE_boolean) +DEFINE_float = _wrapper(gflags.DEFINE_float) +DEFINE_enum = _wrapper(gflags.DEFINE_enum) +DEFINE_list = _wrapper(gflags.DEFINE_list) +DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) +DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) +DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) def DECLARE(name, module_string, flag_values=FLAGS): From e9531ba533ee229f54f5b4369692443113f9e50f Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Wed, 4 Aug 2010 11:14:11 +0100 Subject: [PATCH 09/64] Added note to README. --- xenapi/README | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xenapi/README b/xenapi/README index 1fc67aa7..fbd47103 100644 --- a/xenapi/README +++ b/xenapi/README @@ -1,2 +1,6 @@ This directory contains files that are required for the XenAPI support. They should be installed in the XenServer / Xen Cloud Platform domain 0. + +Also, you need to + +chmod u+x /etc/xapi.d/plugins/objectstore From e39c27fe69fc019a84a2b32ca4ed9f2c1fad9a45 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Wed, 4 Aug 2010 23:45:41 +0100 Subject: [PATCH 10/64] Move the xenapi top level directory under plugins, as suggested by Jay Pipes. --- xenapi/README | 6 - xenapi/etc/xapi.d/plugins/objectstore | 231 -------------------- xenapi/etc/xapi.d/plugins/pluginlib_nova.py | 216 ------------------ 3 files changed, 453 deletions(-) delete mode 100644 xenapi/README delete mode 100644 xenapi/etc/xapi.d/plugins/objectstore delete mode 100755 xenapi/etc/xapi.d/plugins/pluginlib_nova.py diff --git a/xenapi/README b/xenapi/README deleted file mode 100644 index fbd47103..00000000 --- a/xenapi/README +++ /dev/null @@ -1,6 +0,0 @@ -This directory contains files that are required for the XenAPI support. They -should be installed in the XenServer / Xen Cloud Platform domain 0. - -Also, you need to - -chmod u+x /etc/xapi.d/plugins/objectstore diff --git a/xenapi/etc/xapi.d/plugins/objectstore b/xenapi/etc/xapi.d/plugins/objectstore deleted file mode 100644 index 271e7337..00000000 --- a/xenapi/etc/xapi.d/plugins/objectstore +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2010 Citrix Systems, Inc. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# -# XenAPI plugin for fetching images from nova-objectstore. -# - -import base64 -import errno -import hmac -import os -import os.path -import sha -import time -import urlparse - -import XenAPIPlugin - -from pluginlib_nova import * -configure_logging('objectstore') - - -KERNEL_DIR = '/boot/guest' - -DOWNLOAD_CHUNK_SIZE = 2 * 1024 * 1024 -SECTOR_SIZE = 512 -MBR_SIZE_SECTORS = 63 -MBR_SIZE_BYTES = MBR_SIZE_SECTORS * SECTOR_SIZE - - -def get_vdi(session, args): - src_url = exists(args, 'src_url') - username = exists(args, 'username') - password = exists(args, 'password') - add_partition = validate_bool(args, 'add_partition', 'false') - - (proto, netloc, url_path, _, _, _) = urlparse.urlparse(src_url) - - sr = find_sr(session) - if sr is None: - raise Exception('Cannot find SR to write VDI to') - - virtual_size = \ - get_content_length(proto, netloc, url_path, username, password) - if virtual_size < 0: - raise Exception('Cannot get VDI size') - - vdi_size = virtual_size - if add_partition: - # Make room for MBR. - vdi_size += MBR_SIZE_BYTES - - vdi = create_vdi(session, sr, src_url, vdi_size, False) - with_vdi_in_dom0(session, vdi, False, - lambda dev: get_vdi_(proto, netloc, url_path, - username, password, add_partition, - virtual_size, '/dev/%s' % dev)) - return session.xenapi.VDI.get_uuid(vdi) - - -def get_vdi_(proto, netloc, url_path, username, password, add_partition, - virtual_size, dest): - - if add_partition: - write_partition(virtual_size, dest) - - offset = add_partition and MBR_SIZE_BYTES or 0 - get(proto, netloc, url_path, username, password, dest, offset) - - -def write_partition(virtual_size, dest): - mbr_last = MBR_SIZE_SECTORS - 1 - primary_first = MBR_SIZE_SECTORS - primary_last = MBR_SIZE_SECTORS + (virtual_size / SECTOR_SIZE) - 1 - - logging.debug('Writing partition table %d %d to %s...', - primary_first, primary_last, dest) - - result = os.system('parted --script %s mklabel msdos' % dest) - if result != 0: - raise Exception('Failed to mklabel') - result = os.system('parted --script %s mkpart primary %ds %ds' % - (dest, primary_first, primary_last)) - if result != 0: - raise Exception('Failed to mkpart') - - logging.debug('Writing partition table %s done.', dest) - - -def find_sr(session): - host = get_this_host(session) - srs = session.xenapi.SR.get_all() - for sr in srs: - sr_rec = session.xenapi.SR.get_record(sr) - if not ('i18n-key' in sr_rec['other_config'] and - sr_rec['other_config']['i18n-key'] == 'local-storage'): - continue - for pbd in sr_rec['PBDs']: - pbd_rec = session.xenapi.PBD.get_record(pbd) - if pbd_rec['host'] == host: - return sr - return None - - -def get_kernel(session, args): - src_url = exists(args, 'src_url') - username = exists(args, 'username') - password = exists(args, 'password') - - (proto, netloc, url_path, _, _, _) = urlparse.urlparse(src_url) - - dest = os.path.join(KERNEL_DIR, url_path[1:]) - - # Paranoid check against people using ../ to do rude things. - if os.path.commonprefix([KERNEL_DIR, dest]) != KERNEL_DIR: - raise Exception('Illegal destination %s %s', (url_path, dest)) - - dirname = os.path.dirname(dest) - try: - os.makedirs(dirname) - except os.error, e: - if e.errno != errno.EEXIST: - raise - if not os.path.isdir(dirname): - raise Exception('Cannot make directory %s', dirname) - - try: - os.remove(dest) - except: - pass - - get(proto, netloc, url_path, username, password, dest, 0) - - return dest - - -def get_content_length(proto, netloc, url_path, username, password): - headers = make_headers('HEAD', url_path, username, password) - return with_http_connection( - proto, netloc, - lambda conn: get_content_length_(url_path, headers, conn)) - - -def get_content_length_(url_path, headers, conn): - conn.request('HEAD', url_path, None, headers) - response = conn.getresponse() - if response.status != 200: - raise Exception('%d %s' % (response.status, response.reason)) - - return long(response.getheader('Content-Length', -1)) - - -def get(proto, netloc, url_path, username, password, dest, offset): - headers = make_headers('GET', url_path, username, password) - download(proto, netloc, url_path, headers, dest, offset) - - -def make_headers(verb, url_path, username, password): - headers = {} - headers['Date'] = \ - time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) - headers['Authorization'] = \ - 'AWS %s:%s' % (username, - s3_authorization(verb, url_path, password, headers)) - return headers - - -def s3_authorization(verb, path, password, headers): - sha1 = hmac.new(password, digestmod=sha) - sha1.update(plaintext(verb, path, headers)) - return base64.encodestring(sha1.digest()).strip() - - -def plaintext(verb, path, headers): - return '%s\n\n\n%s\n%s' % (verb, - "\n".join([headers[h] for h in headers]), - path) - - -def download(proto, netloc, url_path, headers, dest, offset): - with_http_connection( - proto, netloc, - lambda conn: download_(url_path, dest, offset, headers, conn)) - - -def download_(url_path, dest, offset, headers, conn): - conn.request('GET', url_path, None, headers) - response = conn.getresponse() - if response.status != 200: - raise Exception('%d %s' % (response.status, response.reason)) - - length = response.getheader('Content-Length', -1) - - with_file( - dest, 'a', - lambda dest_file: download_all(response, length, dest_file, offset)) - - -def download_all(response, length, dest_file, offset): - dest_file.seek(offset) - i = 0 - while True: - buf = response.read(DOWNLOAD_CHUNK_SIZE) - if buf: - dest_file.write(buf) - else: - return - i += len(buf) - if length != -1 and i >= length: - return - - -if __name__ == '__main__': - XenAPIPlugin.dispatch({'get_vdi': get_vdi, - 'get_kernel': get_kernel}) diff --git a/xenapi/etc/xapi.d/plugins/pluginlib_nova.py b/xenapi/etc/xapi.d/plugins/pluginlib_nova.py deleted file mode 100755 index 2d323a01..00000000 --- a/xenapi/etc/xapi.d/plugins/pluginlib_nova.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright (c) 2010 Citrix Systems, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# -# Helper functions for the Nova xapi plugins. In time, this will merge -# with the pluginlib.py shipped with xapi, but for now, that file is not -# very stable, so it's easiest just to have a copy of all the functions -# that we need. -# - -import httplib -import logging -import logging.handlers -import re -import time - - -##### Logging setup - -def configure_logging(name): - log = logging.getLogger() - log.setLevel(logging.DEBUG) - sysh = logging.handlers.SysLogHandler('/dev/log') - sysh.setLevel(logging.DEBUG) - formatter = logging.Formatter('%s: %%(levelname)-8s %%(message)s' % name) - sysh.setFormatter(formatter) - log.addHandler(sysh) - - -##### Exceptions - -class PluginError(Exception): - """Base Exception class for all plugin errors.""" - def __init__(self, *args): - Exception.__init__(self, *args) - -class ArgumentError(PluginError): - """Raised when required arguments are missing, argument values are invalid, - or incompatible arguments are given. - """ - def __init__(self, *args): - PluginError.__init__(self, *args) - - -##### Helpers - -def ignore_failure(func, *args, **kwargs): - try: - return func(*args, **kwargs) - except XenAPI.Failure, e: - logging.error('Ignoring XenAPI.Failure %s', e) - return None - - -##### Argument validation - -ARGUMENT_PATTERN = re.compile(r'^[a-zA-Z0-9_:\.\-,]+$') - -def validate_exists(args, key, default=None): - """Validates that a string argument to a RPC method call is given, and - matches the shell-safe regex, with an optional default value in case it - does not exist. - - Returns the string. - """ - if key in args: - if len(args[key]) == 0: - raise ArgumentError('Argument %r value %r is too short.' % (key, args[key])) - if not ARGUMENT_PATTERN.match(args[key]): - raise ArgumentError('Argument %r value %r contains invalid characters.' % (key, args[key])) - if args[key][0] == '-': - raise ArgumentError('Argument %r value %r starts with a hyphen.' % (key, args[key])) - return args[key] - elif default is not None: - return default - else: - raise ArgumentError('Argument %s is required.' % key) - -def validate_bool(args, key, default=None): - """Validates that a string argument to a RPC method call is a boolean string, - with an optional default value in case it does not exist. - - Returns the python boolean value. - """ - value = validate_exists(args, key, default) - if value.lower() == 'true': - return True - elif value.lower() == 'false': - return False - else: - raise ArgumentError("Argument %s may not take value %r. Valid values are ['true', 'false']." % (key, value)) - -def exists(args, key): - """Validates that a freeform string argument to a RPC method call is given. - Returns the string. - """ - if key in args: - return args[key] - else: - raise ArgumentError('Argument %s is required.' % key) - -def optional(args, key): - """If the given key is in args, return the corresponding value, otherwise - return None""" - return key in args and args[key] or None - - -def get_this_host(session): - return session.xenapi.session.get_this_host(session.handle) - - -def get_domain_0(session): - this_host_ref = get_this_host(session) - expr = 'field "is_control_domain" = "true" and field "resident_on" = "%s"' % this_host_ref - return session.xenapi.VM.get_all_records_where(expr).keys()[0] - - -def create_vdi(session, sr_ref, name_label, virtual_size, read_only): - vdi_ref = session.xenapi.VDI.create( - { 'name_label': name_label, - 'name_description': '', - 'SR': sr_ref, - 'virtual_size': str(virtual_size), - 'type': 'User', - 'sharable': False, - 'read_only': read_only, - 'xenstore_data': {}, - 'other_config': {}, - 'sm_config': {}, - 'tags': [] }) - logging.debug('Created VDI %s (%s, %s, %s) on %s.', vdi_ref, name_label, - virtual_size, read_only, sr_ref) - return vdi_ref - - -def with_vdi_in_dom0(session, vdi, read_only, f): - dom0 = get_domain_0(session) - vbd_rec = {} - vbd_rec['VM'] = dom0 - vbd_rec['VDI'] = vdi - vbd_rec['userdevice'] = 'autodetect' - vbd_rec['bootable'] = False - vbd_rec['mode'] = read_only and 'RO' or 'RW' - vbd_rec['type'] = 'disk' - vbd_rec['unpluggable'] = True - vbd_rec['empty'] = False - vbd_rec['other_config'] = {} - vbd_rec['qos_algorithm_type'] = '' - vbd_rec['qos_algorithm_params'] = {} - vbd_rec['qos_supported_algorithms'] = [] - logging.debug('Creating VBD for VDI %s ... ', vdi) - vbd = session.xenapi.VBD.create(vbd_rec) - logging.debug('Creating VBD for VDI %s done.', vdi) - try: - logging.debug('Plugging VBD %s ... ', vbd) - session.xenapi.VBD.plug(vbd) - logging.debug('Plugging VBD %s done.', vbd) - return f(session.xenapi.VBD.get_device(vbd)) - finally: - logging.debug('Destroying VBD for VDI %s ... ', vdi) - vbd_unplug_with_retry(session, vbd) - ignore_failure(session.xenapi.VBD.destroy, vbd) - logging.debug('Destroying VBD for VDI %s done.', vdi) - - -def vbd_unplug_with_retry(session, vbd): - """Call VBD.unplug on the given VBD, with a retry if we get - DEVICE_DETACH_REJECTED. For reasons which I don't understand, we're - seeing the device still in use, even when all processes using the device - should be dead.""" - while True: - try: - session.xenapi.VBD.unplug(vbd) - logging.debug('VBD.unplug successful first time.') - return - except XenAPI.Failure, e: - if (len(e.details) > 0 and - e.details[0] == 'DEVICE_DETACH_REJECTED'): - logging.debug('VBD.unplug rejected: retrying...') - time.sleep(1) - elif (len(e.details) > 0 and - e.details[0] == 'DEVICE_ALREADY_DETACHED'): - logging.debug('VBD.unplug successful eventually.') - return - else: - logging.error('Ignoring XenAPI.Failure in VBD.unplug: %s', e) - return - - -def with_http_connection(proto, netloc, f): - conn = (proto == 'https' and - httplib.HTTPSConnection(netloc) or - httplib.HTTPConnection(netloc)) - try: - return f(conn) - finally: - conn.close() - - -def with_file(dest_path, mode, f): - dest = open(dest_path, mode) - try: - return f(dest) - finally: - dest.close() From a21a29c4403a48dc0181105cd1378e7e940456d7 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 5 Aug 2010 04:31:21 -0700 Subject: [PATCH 11/64] Changed volumes to use a pool instead of globbing filesystem for concurrency reasons. Fixed broken tests. --- nova/tests/volume_unittest.py | 77 +++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 0f4f0e34..2a07afe6 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -17,6 +17,10 @@ # under the License. import logging +import shutil +import tempfile + +from twisted.internet import defer from nova import compute from nova import exception @@ -34,10 +38,16 @@ class VolumeTestCase(test.TrialTestCase): super(VolumeTestCase, self).setUp() self.compute = compute.service.ComputeService() self.volume = None + self.tempdir = tempfile.mkdtemp() self.flags(connection_type='fake', - fake_storage=True) + fake_storage=True, + aoe_export_dir=self.tempdir) self.volume = volume_service.VolumeService() + def tearDown(self): + shutil.rmtree(self.tempdir) + + @defer.inlineCallbacks def test_run_create_volume(self): vol_size = '0' user_id = 'fake' @@ -48,34 +58,40 @@ class VolumeTestCase(test.TrialTestCase): volume_service.get_volume(volume_id)['volume_id']) rv = self.volume.delete_volume(volume_id) - self.assertFailure(volume_service.get_volume(volume_id), - exception.Error) + self.assertRaises(exception.Error, volume_service.get_volume, volume_id) + @defer.inlineCallbacks def test_too_big_volume(self): vol_size = '1001' user_id = 'fake' project_id = 'fake' - self.assertRaises(TypeError, - self.volume.create_volume, - vol_size, user_id, project_id) + try: + yield self.volume.create_volume(vol_size, user_id, project_id) + self.fail("Should have thrown TypeError") + except TypeError: + pass + @defer.inlineCallbacks def test_too_many_volumes(self): vol_size = '1' user_id = 'fake' project_id = 'fake' num_shelves = FLAGS.last_shelf_id - FLAGS.first_shelf_id + 1 - total_slots = FLAGS.slots_per_shelf * num_shelves + total_slots = FLAGS.blades_per_shelf * num_shelves vols = [] + from nova import datastore + redis = datastore.Redis.instance() for i in xrange(total_slots): vid = yield self.volume.create_volume(vol_size, user_id, project_id) vols.append(vid) self.assertFailure(self.volume.create_volume(vol_size, user_id, project_id), - volume_service.NoMoreVolumes) + volume_service.NoMoreBlades) for id in vols: yield self.volume.delete_volume(id) + @defer.inlineCallbacks def test_run_attach_detach_volume(self): # Create one volume and one compute to test with instance_id = "storage-test" @@ -84,22 +100,26 @@ class VolumeTestCase(test.TrialTestCase): project_id = 'fake' mountpoint = "/dev/sdf" volume_id = yield self.volume.create_volume(vol_size, user_id, project_id) - volume_obj = volume_service.get_volume(volume_id) volume_obj.start_attach(instance_id, mountpoint) - rv = yield self.compute.attach_volume(volume_id, - instance_id, - mountpoint) + if FLAGS.fake_tests: + volume_obj.finish_attach() + else: + rv = yield self.compute.attach_volume(instance_id, + volume_id, + mountpoint) self.assertEqual(volume_obj['status'], "in-use") - self.assertEqual(volume_obj['attachStatus'], "attached") + self.assertEqual(volume_obj['attach_status'], "attached") self.assertEqual(volume_obj['instance_id'], instance_id) self.assertEqual(volume_obj['mountpoint'], mountpoint) - self.assertRaises(exception.Error, - self.volume.delete_volume, - volume_id) - - rv = yield self.volume.detach_volume(volume_id) + self.assertFailure(self.volume.delete_volume(volume_id), exception.Error) + volume_obj.start_detach() + if FLAGS.fake_tests: + volume_obj.finish_detach() + else: + rv = yield self.volume.detach_volume(instance_id, + volume_id) volume_obj = volume_service.get_volume(volume_id) self.assertEqual(volume_obj['status'], "available") @@ -108,6 +128,27 @@ class VolumeTestCase(test.TrialTestCase): volume_service.get_volume, volume_id) + @defer.inlineCallbacks + def test_multiple_volume_race_condition(self): + vol_size = "5" + user_id = "fake" + project_id = 'fake' + shelf_blades = [] + def _check(volume_id): + vol = volume_service.get_volume(volume_id) + shelf_blade = '%s.%s' % (vol['shelf_id'], vol['blade_id']) + self.assert_(shelf_blade not in shelf_blades) + shelf_blades.append(shelf_blade) + logging.debug("got %s" % shelf_blade) + vol.destroy() + deferreds = [] + for i in range(5): + d = self.volume.create_volume(vol_size, user_id, project_id) + d.addCallback(_check) + d.addErrback(self.fail) + deferreds.append(d) + yield defer.DeferredList(deferreds) + def test_multi_node(self): # TODO(termie): Figure out how to test with two nodes, # each of them having a different FLAG for storage_node From 1d5e89286da838c3164b994319268b7744448f9f Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 5 Aug 2010 22:54:08 -0400 Subject: [PATCH 12/64] Use webob to simplify wsgi middleware --- nova/endpoint/new_wsgi.py | 122 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 nova/endpoint/new_wsgi.py diff --git a/nova/endpoint/new_wsgi.py b/nova/endpoint/new_wsgi.py new file mode 100644 index 00000000..c7eee20f --- /dev/null +++ b/nova/endpoint/new_wsgi.py @@ -0,0 +1,122 @@ +import eventlet +import eventlet.wsgi +eventlet.patcher.monkey_patch(all=False, socket=True) +import carrot.connection +import carrot.messaging +import itertools +import routes + +# See http://pythonpaste.org/webob/ for usage +from webob.dec import wsgify +from webob import exc, Request, Response + +class WSGILayer(object): + def __init__(self, application=None): + self.application = application + + def __call__(self, environ, start_response): + # Subclasses will probably want to implement __call__ like this: + # + # @wsgify + # def __call__(self, req): + # # Any of the following objects work as responses: + # + # # Option 1: simple string + # resp = 'message\n' + # + # # Option 2: a nicely formatted HTTP exception page + # resp = exc.HTTPForbidden(detail='Nice try') + # + # # Option 3: a webob Response object (in case you need to play with + # # headers, or you want to be treated like an iterable, or or or) + # resp = Response(); resp.app_iter = open('somefile') + # + # # Option 4: any wsgi app to be run next + # resp = self.application + # + # # Option 5: you can get a Response object for a wsgi app, too, to + # # play with headers etc + # resp = req.get_response(self.application) + # + # + # # You can then just return your response... + # return resp # option 1 + # # ... or set req.response and return None. + # req.response = resp # option 2 + # + # See the end of http://pythonpaste.org/webob/modules/dec.html + # for more info. + raise NotImplementedError("You must implement __call__") + + +class Debug(WSGILayer): + @wsgify + def __call__(self, req): + for k, v in req.environ.items(): + print k, "=", v + return self.application + +class Auth(WSGILayer): + @wsgify + def __call__(self, req): + if not 'openstack.auth.token' in req.environ: + # Check auth params here + if True: + req.environ['openstack.auth.token'] = '12345' + else: + return exc.HTTPForbidden(detail="Go away") + + response = req.get_response(self.application) + response.headers['X-Openstack-Auth'] = 'Success' + return response + +class Router(WSGILayer): + def __init__(self, application=None): + super(Router, self).__init__(application) + self.map = routes.Mapper() + self._connect() + + @wsgify + def __call__(self, req): + match = self.map.match(req.path_info) + if match is None: + return self.application + req.environ['openstack.match'] = match + return match['controller'] + + def _connect(self): + raise NotImplementedError("You must implement _connect") + +class FileRouter(Router): + def _connect(self): + self.map.connect(None, '/files/{file}', controller=File()) + self.map.connect(None, '/rfiles/{file}', controller=Reverse(File())) + +class Message(WSGILayer): + @wsgify + def __call__(self, req): + return 'message\n' + +class Reverse(WSGILayer): + @wsgify + def __call__(self, req): + inner_resp = req.get_response(self.application) + print "+" * 80 + Debug()(req) + print "*" * 80 + resp = Response() + resp.app_iter = itertools.imap(lambda x: x[::-1], inner_resp.app_iter) + return resp + +class File(WSGILayer): + @wsgify + def __call__(self, req): + try: + myfile = open(req.environ['openstack.match']['file']) + except IOError, e: + raise exc.HTTPNotFound() + req.response = Response() + req.response.app_iter = myfile + +sock = eventlet.listen(('localhost', 12345)) +eventlet.wsgi.server(sock, Debug(Auth(FileRouter(Message())))) From f193f7b95f51031cc3a1100d13a0750d411f5bdf Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 5 Aug 2010 23:35:16 -0400 Subject: [PATCH 13/64] WsgiStack class, eventletserver.serve. Trying to work toward a simple API that anyone can use to start an eventlet-based server composed of several WSGI apps. --- nova/endpoint/eventletserver.py | 7 +++++++ nova/endpoint/new_wsgi.py | 30 ++++++++++++++++++++++-------- nova/endpoint/rackspace.py | 2 -- 3 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 nova/endpoint/eventletserver.py diff --git a/nova/endpoint/eventletserver.py b/nova/endpoint/eventletserver.py new file mode 100644 index 00000000..b8c15ff5 --- /dev/null +++ b/nova/endpoint/eventletserver.py @@ -0,0 +1,7 @@ +import eventlet +import eventlet.wsgi +eventlet.patcher.monkey_patch(all=False, socket=True) + +def serve(app, port): + sock = eventlet.listen(('0.0.0.0', port)) + eventlet.wsgi.server(sock, app) diff --git a/nova/endpoint/new_wsgi.py b/nova/endpoint/new_wsgi.py index c7eee20f..0f096ddb 100644 --- a/nova/endpoint/new_wsgi.py +++ b/nova/endpoint/new_wsgi.py @@ -1,11 +1,10 @@ -import eventlet -import eventlet.wsgi -eventlet.patcher.monkey_patch(all=False, socket=True) +import eventletserver import carrot.connection import carrot.messaging import itertools import routes + # See http://pythonpaste.org/webob/ for usage from webob.dec import wsgify from webob import exc, Request, Response @@ -49,6 +48,19 @@ class WSGILayer(object): raise NotImplementedError("You must implement __call__") +class WsgiStack(WSGILayer): + def __init__(self, wsgi_layers): + bottom_up = list(reversed(wsgi_layers)) + app, remaining = bottom_up[0], bottom_up[1:] + for layer in remaining: + layer.application = app + app = layer + super(WsgiStack, self).__init__(app) + + @wsgify + def __call__(self, req): + return self.application + class Debug(WSGILayer): @wsgify def __call__(self, req): @@ -101,9 +113,6 @@ class Reverse(WSGILayer): @wsgify def __call__(self, req): inner_resp = req.get_response(self.application) - print "+" * 80 - Debug()(req) - print "*" * 80 resp = Response() resp.app_iter = itertools.imap(lambda x: x[::-1], inner_resp.app_iter) return resp @@ -118,5 +127,10 @@ class File(WSGILayer): req.response = Response() req.response.app_iter = myfile -sock = eventlet.listen(('localhost', 12345)) -eventlet.wsgi.server(sock, Debug(Auth(FileRouter(Message())))) +wsgi_layers = [ + Auth(), + Debug(), + FileRouter(), + Message(), + ] +eventletserver.serve(app=WsgiStack(wsgi_layers), port=12345) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 870aa062..323032eb 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -26,8 +26,6 @@ import logging import multiprocessing import os import time -import tornado.web -from twisted.internet import defer from nova import datastore from nova import exception From 465323b5366a9f25b39a3b687dd141ba83b56343 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Fri, 6 Aug 2010 18:06:57 -0700 Subject: [PATCH 14/64] Changed the network imports to use new network layout. --- nova/endpoint/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 323032eb..7a3fbe14 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -34,7 +34,7 @@ from nova import rpc from nova import utils from nova.auth import manager from nova.compute import model -from nova.compute import network +from nova.network import model as network from nova.endpoint import images from nova.endpoint import wsgi From cf77a0c2370fc37c5e7eba80c7a04cbe517694e2 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Sat, 7 Aug 2010 12:12:10 -0700 Subject: [PATCH 15/64] Reworked WSGI helper module and converted rackspace API endpoint to use it. --- bin/nova-rsapi | 21 +-- nova/endpoint/eventletserver.py | 7 - nova/endpoint/new_wsgi.py | 136 --------------- nova/endpoint/rackspace.py | 286 ++++++++++---------------------- nova/endpoint/wsgi.py | 40 ----- 5 files changed, 90 insertions(+), 400 deletions(-) delete mode 100644 nova/endpoint/eventletserver.py delete mode 100644 nova/endpoint/new_wsgi.py delete mode 100644 nova/endpoint/wsgi.py diff --git a/bin/nova-rsapi b/bin/nova-rsapi index a17efccc..026880d5 100755 --- a/bin/nova-rsapi +++ b/bin/nova-rsapi @@ -1,4 +1,5 @@ #!/usr/bin/env python +# pylint: disable-msg=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the @@ -17,31 +18,17 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - WSGI daemon for the main API endpoint. + Daemon for the Rackspace API endpoint. """ -import logging -from tornado import ioloop -from wsgiref import simple_server - from nova import flags -from nova import rpc -from nova import server from nova import utils -from nova.auth import manager +from nova import wsgi from nova.endpoint import rackspace FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') -def main(_argv): - api_instance = rackspace.Api() - http_server = simple_server.WSGIServer(('0.0.0.0', FLAGS.cc_port), simple_server.WSGIRequestHandler) - http_server.set_app(api_instance.handler) - logging.debug('Started HTTP server on port %i' % FLAGS.cc_port) - while True: - http_server.handle_request() - if __name__ == '__main__': utils.default_flagfile() - server.serve('nova-rsapi', main) + wsgi.run_server(rackspace.API(), FLAGS.cc_port) diff --git a/nova/endpoint/eventletserver.py b/nova/endpoint/eventletserver.py deleted file mode 100644 index b8c15ff5..00000000 --- a/nova/endpoint/eventletserver.py +++ /dev/null @@ -1,7 +0,0 @@ -import eventlet -import eventlet.wsgi -eventlet.patcher.monkey_patch(all=False, socket=True) - -def serve(app, port): - sock = eventlet.listen(('0.0.0.0', port)) - eventlet.wsgi.server(sock, app) diff --git a/nova/endpoint/new_wsgi.py b/nova/endpoint/new_wsgi.py deleted file mode 100644 index 0f096ddb..00000000 --- a/nova/endpoint/new_wsgi.py +++ /dev/null @@ -1,136 +0,0 @@ -import eventletserver -import carrot.connection -import carrot.messaging -import itertools -import routes - - -# See http://pythonpaste.org/webob/ for usage -from webob.dec import wsgify -from webob import exc, Request, Response - -class WSGILayer(object): - def __init__(self, application=None): - self.application = application - - def __call__(self, environ, start_response): - # Subclasses will probably want to implement __call__ like this: - # - # @wsgify - # def __call__(self, req): - # # Any of the following objects work as responses: - # - # # Option 1: simple string - # resp = 'message\n' - # - # # Option 2: a nicely formatted HTTP exception page - # resp = exc.HTTPForbidden(detail='Nice try') - # - # # Option 3: a webob Response object (in case you need to play with - # # headers, or you want to be treated like an iterable, or or or) - # resp = Response(); resp.app_iter = open('somefile') - # - # # Option 4: any wsgi app to be run next - # resp = self.application - # - # # Option 5: you can get a Response object for a wsgi app, too, to - # # play with headers etc - # resp = req.get_response(self.application) - # - # - # # You can then just return your response... - # return resp # option 1 - # # ... or set req.response and return None. - # req.response = resp # option 2 - # - # See the end of http://pythonpaste.org/webob/modules/dec.html - # for more info. - raise NotImplementedError("You must implement __call__") - - -class WsgiStack(WSGILayer): - def __init__(self, wsgi_layers): - bottom_up = list(reversed(wsgi_layers)) - app, remaining = bottom_up[0], bottom_up[1:] - for layer in remaining: - layer.application = app - app = layer - super(WsgiStack, self).__init__(app) - - @wsgify - def __call__(self, req): - return self.application - -class Debug(WSGILayer): - @wsgify - def __call__(self, req): - for k, v in req.environ.items(): - print k, "=", v - return self.application - -class Auth(WSGILayer): - @wsgify - def __call__(self, req): - if not 'openstack.auth.token' in req.environ: - # Check auth params here - if True: - req.environ['openstack.auth.token'] = '12345' - else: - return exc.HTTPForbidden(detail="Go away") - - response = req.get_response(self.application) - response.headers['X-Openstack-Auth'] = 'Success' - return response - -class Router(WSGILayer): - def __init__(self, application=None): - super(Router, self).__init__(application) - self.map = routes.Mapper() - self._connect() - - @wsgify - def __call__(self, req): - match = self.map.match(req.path_info) - if match is None: - return self.application - req.environ['openstack.match'] = match - return match['controller'] - - def _connect(self): - raise NotImplementedError("You must implement _connect") - -class FileRouter(Router): - def _connect(self): - self.map.connect(None, '/files/{file}', controller=File()) - self.map.connect(None, '/rfiles/{file}', controller=Reverse(File())) - -class Message(WSGILayer): - @wsgify - def __call__(self, req): - return 'message\n' - -class Reverse(WSGILayer): - @wsgify - def __call__(self, req): - inner_resp = req.get_response(self.application) - resp = Response() - resp.app_iter = itertools.imap(lambda x: x[::-1], inner_resp.app_iter) - return resp - -class File(WSGILayer): - @wsgify - def __call__(self, req): - try: - myfile = open(req.environ['openstack.match']['file']) - except IOError, e: - raise exc.HTTPNotFound() - req.response = Response() - req.response.app_iter = myfile - -wsgi_layers = [ - Auth(), - Debug(), - FileRouter(), - Message(), - ] -eventletserver.serve(app=WsgiStack(wsgi_layers), port=12345) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 7a3fbe14..f6735a26 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -17,206 +17,95 @@ # under the License. """ -Rackspace API +Rackspace API Endpoint """ -import base64 import json -import logging -import multiprocessing -import os import time -from nova import datastore -from nova import exception +import webob.dec +import webob.exc + from nova import flags from nova import rpc from nova import utils +from nova import wsgi from nova.auth import manager -from nova.compute import model +from nova.compute import model as compute from nova.network import model as network -from nova.endpoint import images -from nova.endpoint import wsgi FLAGS = flags.FLAGS flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') -class Unauthorized(Exception): - pass - -class NotFound(Exception): - pass - - -class Api(object): +class API(wsgi.Middleware): + """Entry point for all requests.""" def __init__(self): - """build endpoints here""" - self.controllers = { - "v1.0": RackspaceAuthenticationApi(), - "servers": RackspaceCloudServerApi() - } + super(API, self).__init__(Router(webob.exc.HTTPNotFound())) - def handler(self, environ, responder): - """ - This is the entrypoint from wsgi. Read PEP 333 and wsgi.org for - more intormation. The key points are responder is a callback that - needs to run before you return, and takes two arguments, response - code string ("200 OK") and headers (["X-How-Cool-Am-I: Ultra-Suede"]) - and the return value is the body of the response. - """ - environ['nova.context'] = self.build_context(environ) - controller, path = wsgi.Util.route( - environ['PATH_INFO'], - self.controllers - ) - logging.debug("Route %s to %s", str(path), str(controller)) - if not controller: - responder("404 Not Found", []) - return "" - try: - rv = controller.process(path, environ) - if type(rv) is tuple: - responder(rv[0], rv[1]) - rv = rv[2] - else: - responder("200 OK", []) - return rv - except Unauthorized: - responder("401 Unauthorized", []) - return "" - except NotFound: - responder("404 Not Found", []) - return "" + def __call__(self, environ, start_response): + context = {} + if "HTTP_X_AUTH_TOKEN" in environ: + context['user'] = manager.AuthManager().get_user_from_access_key( + environ['HTTP_X_AUTH_TOKEN']) + if context['user']: + context['project'] = manager.AuthManager().get_project( + context['user'].name) + if "user" not in context: + return webob.exc.HTTPForbidden()(environ, start_response) + environ['nova.context'] = context + return self.application(environ, start_response) - def build_context(self, env): - rv = {} - if env.has_key("HTTP_X_AUTH_TOKEN"): - rv['user'] = manager.AuthManager().get_user_from_access_key( - env['HTTP_X_AUTH_TOKEN'] - ) - if rv['user']: - rv['project'] = manager.AuthManager().get_project( - rv['user'].name - ) - return rv +class Router(wsgi.Router): + """Route requests to the next WSGI application.""" + + def _build_map(self): + """Build routing map for authentication and cloud.""" + self._connect("/v1.0", controller=AuthenticationAPI()) + cloud = CloudServerAPI() + self._connect("/servers", controller=cloud.launch_server, + conditions={"method": ["POST"]}) + self._connect("/servers/{server_id}", controller=cloud.delete_server, + conditions={'method': ["DELETE"]}) + self._connect("/servers", controller=cloud) -class RackspaceApiEndpoint(object): - def process(self, path, env): - """ - Main entrypoint for all controllers (what gets run by the wsgi handler). - Check authentication based on key, raise Unauthorized if invalid. +class AuthenticationAPI(wsgi.Application): + """Handle all authorization requests through WSGI applications.""" - Select the most appropriate action based on request type GET, POST, etc, - then pass it through to the implementing controller. Defalut to GET if - the implementing child doesn't respond to a particular type. - """ - if not self.check_authentication(env): - raise Unauthorized("Unable to authenticate") - - method = env['REQUEST_METHOD'].lower() - callback = getattr(self, method, None) - if not callback: - callback = getattr(self, "get") - logging.debug("%s processing %s with %s", self, method, callback) - return callback(path, env) - - def get(self, path, env): - """ - The default GET will look at the path and call an appropriate - action within this controller based on the the structure of the path. - - Given the following path lengths (with the first part stripped of by - router, as it is the controller name): - = 0 -> index - = 1 -> first component (/servers/details -> details) - >= 2 -> second path component (/servers/ID/ips/* -> ips) - - This should return - A String if 200 OK and no additional headers - (CODE, HEADERS, BODY) for custom response code and headers - """ - if len(path) == 0 and hasattr(self, "index"): - logging.debug("%s running index", self) - return self.index(env) - if len(path) >= 2: - action = path[1] - else: - action = path.pop(0) - - logging.debug("%s running action %s", self, action) - if hasattr(self, action): - method = getattr(self, action) - return method(path, env) - else: - raise NotFound("Missing method %s" % path[0]) - - def check_authentication(self, env): - if not env['nova.context']['user']: - return False - return True + @webob.dec.wsgify + def __call__(self, req): # pylint: disable-msg=W0221 + # TODO(todd): make a actual session with a unique token + # just pass the auth key back through for now + res = webob.Response() + res.status = '204 No Content' + res.headers.add('X-Server-Management-Url', req.host_url) + res.headers.add('X-Storage-Url', req.host_url) + res.headers.add('X-CDN-Managment-Url', req.host_url) + res.headers.add('X-Auth-Token', req.headers['X-Auth-Key']) + return res -class RackspaceAuthenticationApi(object): - - def process(self, path, env): - return self.index(path, env) - - # TODO(todd): make a actual session with a unique token - # just pass the auth key back through for now - def index(self, _path, env): - response = '204 No Content' - headers = [ - ('X-Server-Management-Url', 'http://%s' % env['HTTP_HOST']), - ('X-Storage-Url', 'http://%s' % env['HTTP_HOST']), - ('X-CDN-Managment-Url', 'http://%s' % env['HTTP_HOST']), - ('X-Auth-Token', env['HTTP_X_AUTH_KEY']) - ] - body = "" - return (response, headers, body) - - -class RackspaceCloudServerApi(RackspaceApiEndpoint): +class CloudServerAPI(wsgi.Application): + """Handle all server requests through WSGI applications.""" def __init__(self): - self.instdir = model.InstanceDirectory() + super(CloudServerAPI, self).__init__() + self.instdir = compute.InstanceDirectory() self.network = network.PublicNetworkController() - def post(self, path, env): - if len(path) == 0: - return self.launch_server(env) - - def delete(self, path_parts, env): - if self.delete_server(path_parts[0]): - return ("202 Accepted", [], "") - else: - return ("404 Not Found", [], - "Did not find image, or it was not in a running state") - - - def index(self, env): - return self.detail(env) - - def detail(self, args, env): + @webob.dec.wsgify + def __call__(self, req): # pylint: disable-msg=W0221 value = {"servers": []} for inst in self.instdir.all: value["servers"].append(self.instance_details(inst)) return json.dumps(value) - ## - ## - - def launch_server(self, env): - data = json.loads(env['wsgi.input'].read(int(env['CONTENT_LENGTH']))) - inst = self.build_server_instance(data, env['nova.context']) - self.schedule_launch_of_instance(inst) - return json.dumps({"server": self.instance_details(inst)}) - - def instance_details(self, inst): + def instance_details(self, inst): # pylint: disable-msg=R0201 + "Build the data structure to represent details for an instance." return { "id": inst.get("instance_id", None), "imageId": inst.get("image_id", None), @@ -224,11 +113,9 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): "hostId": inst.get("node_name", None), "status": inst.get("state", "pending"), "addresses": { - "public": [self.network.get_public_ip_for_instance( - inst.get("instance_id", None) - )], - "private": [inst.get("private_dns_name", None)] - }, + "public": [network.get_public_ip_for_instance( + inst.get("instance_id", None))], + "private": [inst.get("private_dns_name", None)]}, # implemented only by Rackspace, not AWS "name": inst.get("name", "Not-Specified"), @@ -237,11 +124,22 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): "progress": "Not-Supported", "metadata": { "Server Label": "Not-Supported", - "Image Version": "Not-Supported" - } - } + "Image Version": "Not-Supported"}} + + @webob.dec.wsgify + def launch_server(self, req): + """Launch a new instance.""" + data = json.loads(req.body) + inst = self.build_server_instance(data, req.environ['nova.context']) + rpc.cast( + FLAGS.compute_topic, { + "method": "run_instance", + "args": {"instance_id": inst.instance_id}}) + + return json.dumps({"server": self.instance_details(inst)}) def build_server_instance(self, env, context): + """Build instance data structure and save it to the data store.""" reservation = utils.generate_uid('r') ltime = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) inst = self.instdir.new() @@ -253,45 +151,33 @@ class RackspaceCloudServerApi(RackspaceApiEndpoint): inst['reservation_id'] = reservation inst['launch_time'] = ltime inst['mac_address'] = utils.generate_mac() - address = network.allocate_ip( + address = self.network.allocate_ip( inst['user_id'], inst['project_id'], - mac=inst['mac_address'] - ) + mac=inst['mac_address']) inst['private_dns_name'] = str(address) inst['bridge_name'] = network.BridgedNetwork.get_network_for_project( inst['user_id'], inst['project_id'], - 'default' # security group - )['bridge_name'] + 'default')['bridge_name'] # key_data, key_name, ami_launch_index # TODO(todd): key data or root password inst.save() return inst - def schedule_launch_of_instance(self, inst): - rpc.cast( - FLAGS.compute_topic, - { - "method": "run_instance", - "args": {"instance_id": inst.instance_id} - } - ) - - def delete_server(self, instance_id): - owner_hostname = self.host_for_instance(instance_id) - # it isn't launched? + @webob.dec.wsgify + @wsgi.route_args + def delete_server(self, req, route_args): # pylint: disable-msg=R0201 + """Delete an instance.""" + owner_hostname = None + instance = compute.Instance.lookup(route_args['server_id']) + if instance: + owner_hostname = instance["node_name"] if not owner_hostname: - return None + return webob.exc.HTTPNotFound("Did not find image, or it was " + "not in a running state.") rpc_transport = "%s:%s" % (FLAGS.compute_topic, owner_hostname) rpc.cast(rpc_transport, {"method": "reboot_instance", - "args": {"instance_id": instance_id}}) - return True - - def host_for_instance(self, instance_id): - instance = model.Instance.lookup(instance_id) - if not instance: - return None - return instance["node_name"] - + "args": {"instance_id": route_args['server_id']}}) + req.status = "202 Accepted" diff --git a/nova/endpoint/wsgi.py b/nova/endpoint/wsgi.py deleted file mode 100644 index b7bb588c..00000000 --- a/nova/endpoint/wsgi.py +++ /dev/null @@ -1,40 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -''' -Utility methods for working with WSGI servers -''' - -class Util(object): - - @staticmethod - def route(reqstr, controllers): - if len(reqstr) == 0: - return Util.select_root_controller(controllers), [] - parts = [x for x in reqstr.split("/") if len(x) > 0] - if len(parts) == 0: - return Util.select_root_controller(controllers), [] - return controllers[parts[0]], parts[1:] - - @staticmethod - def select_root_controller(controllers): - if '' in controllers: - return controllers[''] - else: - return None - From 288dc19762b86d30ea1637b1cd76db357a2fc11b Mon Sep 17 00:00:00 2001 From: Eric Day Date: Sat, 7 Aug 2010 19:51:17 -0700 Subject: [PATCH 16/64] Cleaned up pep8/pylint for bin/* files. I did not fix rsapi since this is already cleaned up in another branch. --- bin/nova-api | 6 +- bin/nova-dhcpbridge | 42 ++++---- bin/nova-import-canonical-imagestore | 31 +++--- bin/nova-instancemonitor | 18 +--- bin/nova-manage | 138 +++++++++++++++------------ bin/nova-objectstore | 8 +- run_tests.py | 3 +- 7 files changed, 127 insertions(+), 119 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 1f2009c3..13baf22a 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -29,8 +29,6 @@ from nova import flags from nova import rpc from nova import server from nova import utils -from nova.auth import manager -from nova.compute import model from nova.endpoint import admin from nova.endpoint import api from nova.endpoint import cloud @@ -39,10 +37,10 @@ FLAGS = flags.FLAGS def main(_argv): + """Load the controllers and start the tornado I/O loop.""" controllers = { 'Cloud': cloud.CloudController(), - 'Admin': admin.AdminController() - } + 'Admin': admin.AdminController()} _app = api.APIServerApplication(controllers) conn = rpc.Connection.instance() diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index b3e7d456..ed1af206 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -18,8 +18,6 @@ # under the License. """ -nova-dhcpbridge - Handle lease database updates from DHCP servers. """ @@ -42,34 +40,43 @@ from nova.network import service FLAGS = flags.FLAGS -def add_lease(mac, ip, hostname, interface): +def add_lease(_mac, ip, _hostname, _interface): + """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: service.VlanNetworkService().lease_ip(ip) else: - rpc.cast("%s.%s" (FLAGS.network_topic, FLAGS.node_name), + rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "lease_ip", - "args" : {"fixed_ip": ip}}) + "args": {"fixed_ip": ip}}) -def old_lease(mac, ip, hostname, interface): + +def old_lease(_mac, _ip, _hostname, _interface): + """Do nothing, just an old lease update.""" logging.debug("Adopted old lease or got a change of mac/hostname") -def del_lease(mac, ip, hostname, interface): + +def del_lease(_mac, ip, _hostname, _interface): + """Remove the leased IP from the databases.""" if FLAGS.fake_rabbit: service.VlanNetworkService().release_ip(ip) else: - rpc.cast("%s.%s" (FLAGS.network_topic, FLAGS.node_name), + rpc.cast("%s.%s" % (FLAGS.network_topic, FLAGS.node_name), {"method": "release_ip", - "args" : {"fixed_ip": ip}}) + "args": {"fixed_ip": ip}}) + def init_leases(interface): + """Get the list of hosts for an interface.""" net = model.get_network_by_interface(interface) res = "" for host_name in net.hosts: - res += "%s\n" % linux_net.hostDHCP(net, host_name, net.hosts[host_name]) + res += "%s\n" % linux_net.hostDHCP(net, host_name, + net.hosts[host_name]) return res def main(): + """Parse environment and arguments and call the approproate action.""" flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) @@ -79,18 +86,19 @@ def main(): FLAGS.redis_db = 8 FLAGS.network_size = 32 FLAGS.connection_type = 'fake' - FLAGS.fake_network=True - FLAGS.auth_driver='nova.auth.ldapdriver.FakeLdapDriver' + FLAGS.fake_network = True + FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' action = argv[1] - if action in ['add','del','old']: + if action in ['add', 'del', 'old']: mac = argv[2] ip = argv[3] hostname = argv[4] - logging.debug("Called %s for mac %s with ip %s and hostname %s on interface %s" % (action, mac, ip, hostname, interface)) - globals()[action+'_lease'](mac, ip, hostname, interface) + logging.debug("Called %s for mac %s with ip %s and " + "hostname %s on interface %s", + action, mac, ip, hostname, interface) + globals()[action + '_lease'](mac, ip, hostname, interface) else: print init_leases(interface) - exit(0) if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 2e79f09b..5165109b 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -37,20 +37,17 @@ FLAGS = flags.FLAGS api_url = 'https://imagestore.canonical.com/api/dashboard' -image_cache = None -def images(): - global image_cache - if not image_cache: - try: - images = json.load(urllib2.urlopen(api_url))['images'] - image_cache = [i for i in images if i['title'].find('amd64') > -1] - except Exception: - print 'unable to download canonical image list' - sys.exit(1) - return image_cache -# FIXME(ja): add checksum/signature checks +def get_images(): + """Get a list of the images from the imagestore URL.""" + images = json.load(urllib2.urlopen(api_url))['images'] + images = [img for img in images if img['title'].find('amd64') > -1] + return images + + def download(img): + """Download an image to the local filesystem.""" + # FIXME(ja): add checksum/signature checks tempdir = tempfile.mkdtemp(prefix='cis-') kernel_id = None @@ -79,20 +76,22 @@ def download(img): shutil.rmtree(tempdir) + def main(): + """Main entry point.""" utils.default_flagfile() argv = FLAGS(sys.argv) + images = get_images() if len(argv) == 2: - for img in images(): + for img in images: if argv[1] == 'all' or argv[1] == img['title']: download(img) else: print 'usage: %s (title|all)' print 'available images:' - for image in images(): - print image['title'] + for img in images: + print img['title'] if __name__ == '__main__': main() - diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index b195089b..911fb6f4 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -22,7 +22,6 @@ """ import logging -from twisted.internet import task from twisted.application import service from nova import twistd @@ -30,7 +29,11 @@ from nova.compute import monitor logging.getLogger('boto').setLevel(logging.WARN) -def main(): + +if __name__ == '__main__': + twistd.serve(__file__) + +if __name__ == '__builtin__': logging.warn('Starting instance monitor') m = monitor.InstanceMonitor() @@ -38,14 +41,3 @@ def main(): # parses this file, return it so that we can get it into globals below application = service.Application('nova-instancemonitor') m.setServiceParent(application) - return application - -if __name__ == '__main__': - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = main() - - - - diff --git a/bin/nova-manage b/bin/nova-manage index 7835c7a7..2dd569df 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -37,12 +37,15 @@ FLAGS = flags.FLAGS class VpnCommands(object): + """Class for managing VPNs.""" + def __init__(self): self.manager = manager.AuthManager() self.instdir = model.InstanceDirectory() self.pipe = pipelib.CloudPipe(cloud.CloudController()) def list(self): + """Print a listing of the VPNs for all projects.""" print "%-12s\t" % 'project', print "%-12s\t" % 'ip:port', print "%s" % 'state' @@ -50,9 +53,10 @@ class VpnCommands(object): print "%-12s\t" % project.name, print "%s:%s\t" % (project.vpn_ip, project.vpn_port), - vpn = self.__vpn_for(project.id) + vpn = self._vpn_for(project.id) if vpn: - out, err = utils.execute("ping -c1 -w1 %s > /dev/null; echo $?" % vpn['private_dns_name']) + command = "ping -c1 -w1 %s > /dev/null; echo $?" + out, _err = utils.execute(command % vpn['private_dns_name']) if out.strip() == '0': net = 'up' else: @@ -66,25 +70,32 @@ class VpnCommands(object): else: print None - def __vpn_for(self, project_id): + def _vpn_for(self, project_id): + """Get the VPN instance for a project ID.""" for instance in self.instdir.all: - if (instance.state.has_key('image_id') + if ('image_id' in instance.state and instance['image_id'] == FLAGS.vpn_image_id - and not instance['state_description'] in ['shutting_down', 'shutdown'] + and not instance['state_description'] in + ['shutting_down', 'shutdown'] and instance['project_id'] == project_id): return instance def spawn(self): + """Run all VPNs.""" for p in reversed(self.manager.get_projects()): - if not self.__vpn_for(p.id): - print 'spawning %s' % p.id - self.pipe.launch_vpn_instance(p.id) - time.sleep(10) + if not self._vpn_for(p.id): + print 'spawning %s' % p.id + self.pipe.launch_vpn_instance(p.id) + time.sleep(10) def run(self, project_id): + """Start the VPN for a given project.""" self.pipe.launch_vpn_instance(project_id) + class RoleCommands(object): + """Class for managing roles.""" + def __init__(self): self.manager = manager.AuthManager() @@ -107,25 +118,24 @@ class RoleCommands(object): arguments: user, role [project]""" self.manager.remove_role(user, role, project) + class UserCommands(object): + """Class for managing users.""" + def __init__(self): self.manager = manager.AuthManager() - def __print_export(self, user): - print 'export EC2_ACCESS_KEY=%s' % user.access - print 'export EC2_SECRET_KEY=%s' % user.secret - def admin(self, name, access=None, secret=None): """creates a new admin and prints exports arguments: name [access] [secret]""" user = self.manager.create_user(name, access, secret, True) - self.__print_export(user) + print_export(user) def create(self, name, access=None, secret=None): """creates a new user and prints exports arguments: name [access] [secret]""" user = self.manager.create_user(name, access, secret, False) - self.__print_export(user) + print_export(user) def delete(self, name): """deletes an existing user @@ -137,7 +147,7 @@ class UserCommands(object): arguments: name""" user = self.manager.get_user(name) if user: - self.__print_export(user) + print_export(user) else: print "User %s doesn't exist" % name @@ -147,53 +157,58 @@ class UserCommands(object): for user in self.manager.get_users(): print user.name + +def print_export(user): + """Print export variables to use with API.""" + print 'export EC2_ACCESS_KEY=%s' % user.access + print 'export EC2_SECRET_KEY=%s' % user.secret + + class ProjectCommands(object): + """Class for managing projects.""" + def __init__(self): self.manager = manager.AuthManager() def add(self, project, user): - """adds user to project + """Adds user to project arguments: project user""" self.manager.add_to_project(user, project) def create(self, name, project_manager, description=None): - """creates a new project + """Creates a new project arguments: name project_manager [description]""" - user = self.manager.create_project(name, project_manager, description) + self.manager.create_project(name, project_manager, description) def delete(self, name): - """deletes an existing project + """Deletes an existing project arguments: name""" self.manager.delete_project(name) def environment(self, project_id, user_id, filename='novarc'): - """exports environment variables to an sourcable file + """Exports environment variables to an sourcable file arguments: project_id user_id [filename='novarc]""" rc = self.manager.get_environment_rc(project_id, user_id) with open(filename, 'w') as f: f.write(rc) def list(self): - """lists all projects + """Lists all projects arguments: """ for project in self.manager.get_projects(): print project.name def remove(self, project, user): - """removes user from project + """Removes user from project arguments: project user""" self.manager.remove_from_project(user, project) - def zip(self, project_id, user_id, filename='nova.zip'): - """exports credentials for project to a zip file + def create_zip(self, project_id, user_id, filename='nova.zip'): + """Exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" - zip = self.manager.get_credentials(project_id, user_id) + zip_file = self.manager.get_credentials(project_id, user_id) with open(filename, 'w') as f: - f.write(zip) - - -def usage(script_name): - print script_name + " category action []" + f.write(zip_file) categories = [ @@ -205,62 +220,61 @@ categories = [ def lazy_match(name, key_value_tuples): - """finds all objects that have a key that case insensitively contains [name] - key_value_tuples is a list of tuples of the form (key, value) + """Finds all objects that have a key that case insensitively contains + [name] key_value_tuples is a list of tuples of the form (key, value) returns a list of tuples of the form (key, value)""" - return [(k, v) for (k, v) in key_value_tuples if k.lower().find(name.lower()) == 0] + result = [] + for (k, v) in key_value_tuples: + if k.lower().find(name.lower()) == 0: + result.append((k, v)) + if len(result) == 0: + print "%s does not match any options:" % name + for k, _v in key_value_tuples: + print "\t%s" % k + sys.exit(2) + if len(result) > 1: + print "%s matched multiple options:" % name + for k, _v in result: + print "\t%s" % k + sys.exit(2) + return result def methods_of(obj): - """get all callable methods of an object that don't start with underscore + """Get all callable methods of an object that don't start with underscore returns a list of tuples of the form (method_name, method)""" - return [(i, getattr(obj, i)) for i in dir(obj) if callable(getattr(obj, i)) and not i.startswith('_')] + result = [] + for i in dir(obj): + if callable(getattr(obj, i)) and not i.startswith('_'): + result.append((i, getattr(obj, i))) + return result -if __name__ == '__main__': +def main(): + """Parse options and call the appropriate class/method.""" utils.default_flagfile('/etc/nova/nova-manage.conf') argv = FLAGS(sys.argv) script_name = argv.pop(0) if len(argv) < 1: - usage(script_name) + print script_name + " category action []" print "Available categories:" - for k, v in categories: + for k, _ in categories: print "\t%s" % k sys.exit(2) category = argv.pop(0) matches = lazy_match(category, categories) - if len(matches) == 0: - print "%s does not match any categories:" % category - for k, v in categories: - print "\t%s" % k - sys.exit(2) - if len(matches) > 1: - print "%s matched multiple categories:" % category - for k, v in matches: - print "\t%s" % k - sys.exit(2) # instantiate the command group object category, fn = matches[0] command_object = fn() actions = methods_of(command_object) if len(argv) < 1: - usage(script_name) + print script_name + " category action []" print "Available actions for %s category:" % category - for k, v in actions: + for k, _v in actions: print "\t%s" % k sys.exit(2) action = argv.pop(0) matches = lazy_match(action, actions) - if len(matches) == 0: - print "%s does not match any actions" % action - for k, v in actions: - print "\t%s" % k - sys.exit(2) - if len(matches) > 1: - print "%s matched multiple actions:" % action - for k, v in matches: - print "\t%s" % k - sys.exit(2) action, fn = matches[0] # call the action with the remaining arguments try: @@ -271,3 +285,5 @@ if __name__ == '__main__': print "%s %s: %s" % (category, action, fn.__doc__) sys.exit(2) +if __name__ == '__main__': + main() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index c0fa815c..02f2bcb4 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -30,15 +30,9 @@ from nova.objectstore import handler FLAGS = flags.FLAGS -def main(): - app = handler.get_application() - print app - return app - -# NOTE(soren): Stolen from nova-compute if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': utils.default_flagfile() - application = main() + application = handler.get_application() diff --git a/run_tests.py b/run_tests.py index 5a8966f0..f6208825 100644 --- a/run_tests.py +++ b/run_tests.py @@ -68,7 +68,8 @@ flags.DEFINE_bool('flush_db', True, 'Flush the database before running fake tests') flags.DEFINE_string('tests_stderr', 'run_tests.err.log', - 'Path to where to pipe STDERR during test runs. Default = "run_tests.err.log"') + 'Path to where to pipe STDERR during test runs. ' + 'Default = "run_tests.err.log"') if __name__ == '__main__': OptionsClass = twistd.WrapTwistedOptions(trial_script.Options) From 1dbf1505070ea76a740f995021e51c9bbbb3f9bb Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 09:50:22 -0700 Subject: [PATCH 17/64] attempting some cleanup work --- nova/endpoint/cloud.py | 77 ++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 878d54a1..ee22863a 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -47,6 +47,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') + def _gen_key(user_id, key_name): """ Tuck this into AuthManager """ try: @@ -102,15 +103,16 @@ class CloudController(object): result = {} for instance in self.instdir.all: if instance['project_id'] == project_id: - line = '%s slots=%d' % (instance['private_dns_name'], INSTANCE_TYPES[instance['instance_type']]['vcpus']) + line = '%s slots=%d' % (instance['private_dns_name'], + INSTANCE_TYPES[instance['instance_type']]['vcpus']) if instance['key_name'] in result: result[instance['key_name']].append(line) else: result[instance['key_name']] = [line] return result - def get_metadata(self, ip): - i = self.get_instance_by_ip(ip) + def get_metadata(self, ipaddress): + i = self.get_instance_by_ip(ipaddress) if i is None: return None mpi = self._get_mpi_data(i['project_id']) @@ -147,7 +149,7 @@ class CloudController(object): }, 'public-hostname': i.get('dns_name', ''), 'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP - 'public-keys' : keys, + 'public-keys': keys, 'ramdisk-id': i.get('ramdisk_id', ''), 'reservation-id': i['reservation_id'], 'security-groups': i.get('groups', ''), @@ -203,7 +205,7 @@ class CloudController(object): 'keyFingerprint': key_pair.fingerprint, }) - return { 'keypairsSet': result } + return {'keypairsSet': result} @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): @@ -232,7 +234,7 @@ class CloudController(object): @rbac.allow('all') def describe_security_groups(self, context, group_names, **kwargs): - groups = { 'securityGroupSet': [] } + groups = {'securityGroupSet': []} # Stubbed for now to unblock other things. return groups @@ -251,7 +253,7 @@ class CloudController(object): instance = self._get_instance(context, instance_id[0]) return rpc.call('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "get_console_output", - "args" : {"instance_id": instance_id[0]}}) + "args": {"instance_id": instance_id[0]}}) def _get_user_id(self, context): if context and context.user: @@ -285,10 +287,10 @@ class CloudController(object): if volume['attach_status'] == 'attached': v['attachmentSet'] = [{'attachTime': volume['attach_time'], 'deleteOnTermination': volume['delete_on_termination'], - 'device' : volume['mountpoint'], - 'instanceId' : volume['instance_id'], - 'status' : 'attached', - 'volume_id' : volume['volume_id']}] + 'device': volume['mountpoint'], + 'instanceId': volume['instance_id'], + 'status': 'attached', + 'volume_id': volume['volume_id']}] else: v['attachmentSet'] = [{}] return v @@ -298,7 +300,7 @@ class CloudController(object): def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", - "args" : {"size": size, + "args": {"size": size, "user_id": context.user.id, "project_id": context.project.id}}) # NOTE(vish): rpc returned value is in the result key in the dictionary @@ -348,15 +350,15 @@ class CloudController(object): compute_node = instance['node_name'] rpc.cast('%s.%s' % (FLAGS.compute_topic, compute_node), {"method": "attach_volume", - "args" : {"volume_id": volume_id, - "instance_id" : instance_id, - "mountpoint" : device}}) - return defer.succeed({'attachTime' : volume['attach_time'], - 'device' : volume['mountpoint'], - 'instanceId' : instance_id, - 'requestId' : context.request_id, - 'status' : volume['attach_status'], - 'volumeId' : volume_id}) + "args": {"volume_id": volume_id, + "instance_id": instance_id, + "mountpoint": device}}) + return defer.succeed({'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) @rbac.allow('projectmanager', 'sysadmin') @@ -372,18 +374,18 @@ class CloudController(object): instance = self._get_instance(context, instance_id) rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "detach_volume", - "args" : {"instance_id": instance_id, + "args": {"instance_id": instance_id, "volume_id": volume_id}}) except exception.NotFound: # If the instance doesn't exist anymore, # then we need to call detach blind volume.finish_detach() - return defer.succeed({'attachTime' : volume['attach_time'], - 'device' : volume['mountpoint'], - 'instanceId' : instance_id, - 'requestId' : context.request_id, - 'status' : volume['attach_status'], - 'volumeId' : volume_id}) + return defer.succeed({'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) def _convert_to_set(self, lst, label): if lst == None or lst == []: @@ -425,7 +427,8 @@ class CloudController(object): i['key_name'] = instance.get('key_name', None) if context.user.is_admin(): i['key_name'] = '%s (%s, %s)' % (i['key_name'], - instance.get('project_id', None), instance.get('node_name','')) + instance.get('project_id', None), + instance.get('node_name', '')) i['product_codes_set'] = self._convert_to_set( instance.get('product_codes', None), 'product_code') i['instance_type'] = instance.get('instance_type', None) @@ -442,7 +445,7 @@ class CloudController(object): reservations[res_id] = r reservations[res_id]['instances_set'].append(i) - instance_response = {'reservationSet' : list(reservations.values()) } + instance_response = {'reservationSet': list(reservations.values())} return instance_response @rbac.allow('all') @@ -457,7 +460,7 @@ class CloudController(object): address['project_id'] == context.project.id): address_rv = { 'public_ip': address['address'], - 'instance_id' : address.get('instance_id', 'free') + 'instance_id': address.get('instance_id', 'free') } if context.user.is_admin(): address_rv['instance_id'] = "%s (%s, %s)" % ( @@ -477,7 +480,7 @@ class CloudController(object): "args": {"user_id": context.user.id, "project_id": context.project.id}}) public_ip = alloc_result['result'] - defer.returnValue({'addressSet': [{'publicIp' : public_ip}]}) + defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) @rbac.allow('netadmin') @defer.inlineCallbacks @@ -591,7 +594,7 @@ class CloudController(object): inst.save() rpc.cast(FLAGS.compute_topic, {"method": "run_instance", - "args": {"instance_id" : inst.instance_id}}) + "args": {"instance_id": inst.instance_id}}) logging.debug("Casting to node for %s's instance with IP of %s" % (context.user.name, inst['private_dns_name'])) # TODO: Make Network figure out the network name from ip. @@ -646,7 +649,7 @@ class CloudController(object): instance = self._get_instance(context, i) rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "reboot_instance", - "args" : {"instance_id": i}}) + "args": {"instance_id": i}}) return defer.succeed(True) @rbac.allow('projectmanager', 'sysadmin') @@ -656,7 +659,7 @@ class CloudController(object): volume_node = volume['node_name'] rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node), {"method": "delete_volume", - "args" : {"volume_id": volume_id}}) + "args": {"volume_id": volume_id}}) return defer.succeed(True) @rbac.allow('all') @@ -689,9 +692,9 @@ class CloudController(object): image = images.list(context, image_id)[0] except IndexError: raise exception.ApiError('invalid id: %s' % image_id) - result = { 'image_id': image_id, 'launchPermission': [] } + result = {'image_id': image_id, 'launchPermission': []} if image['isPublic']: - result['launchPermission'].append({ 'group': 'all' }) + result['launchPermission'].append({'group': 'all'}) return defer.succeed(result) @rbac.allow('projectmanager', 'sysadmin') From e1595dbe4af36228e63ebf5c761fc40876d5ef3f Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 11:39:14 -0700 Subject: [PATCH 18/64] variable name cleanup --- nova/endpoint/cloud.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index ee22863a..8b937306 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -210,18 +210,18 @@ class CloudController(object): @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): try: - d = defer.Deferred() - p = context.handler.application.settings.get('pool') + dcall = defer.Deferred() + pool = context.handler.application.settings.get('pool') def _complete(kwargs): if 'exception' in kwargs: - d.errback(kwargs['exception']) + dcall.errback(kwargs['exception']) return - d.callback({'keyName': key_name, + dcall.callback({'keyName': key_name, 'keyFingerprint': kwargs['fingerprint'], 'keyMaterial': kwargs['private_key']}) - p.apply_async(_gen_key, [context.user.id, key_name], + pool.apply_async(_gen_key, [context.user.id, key_name], callback=_complete) - return d + return dcall except manager.UserError as e: raise From 97f012cf1e65045b2fce1b6048d9210db98ba996 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 11:40:03 -0700 Subject: [PATCH 19/64] removing what appears to be an unused try/except statement - nova.auth.manager.UserError doesn't exist in this codebase. Leftover? Something intended to be there but never added? --- nova/endpoint/cloud.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 8b937306..ad9188ff 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -209,22 +209,18 @@ class CloudController(object): @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): - try: - dcall = defer.Deferred() - pool = context.handler.application.settings.get('pool') - def _complete(kwargs): - if 'exception' in kwargs: - dcall.errback(kwargs['exception']) - return - dcall.callback({'keyName': key_name, - 'keyFingerprint': kwargs['fingerprint'], - 'keyMaterial': kwargs['private_key']}) - pool.apply_async(_gen_key, [context.user.id, key_name], - callback=_complete) - return dcall - - except manager.UserError as e: - raise + dcall = defer.Deferred() + pool = context.handler.application.settings.get('pool') + def _complete(kwargs): + if 'exception' in kwargs: + dcall.errback(kwargs['exception']) + return + dcall.callback({'keyName': key_name, + 'keyFingerprint': kwargs['fingerprint'], + 'keyMaterial': kwargs['private_key']}) + pool.apply_async(_gen_key, [context.user.id, key_name], + callback=_complete) + return dcall @rbac.allow('all') def delete_key_pair(self, context, key_name, **kwargs): From 476235848836ad825965aee83bac3443e22cdd89 Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Sun, 8 Aug 2010 13:31:40 -0700 Subject: [PATCH 20/64] light cleanup - convention stuff mostly --- nova/auth/manager.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index d44ed52b..e5efbca2 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -29,15 +29,17 @@ import uuid import zipfile from nova import crypto -from nova import datastore from nova import exception from nova import flags -from nova import objectstore # for flags from nova import utils -from nova.auth import ldapdriver # for flags from nova.auth import signer from nova.network import vpn +#unused imports +#from nova import datastore +#from nova.auth import ldapdriver # for flags +#from nova import objectstore # for flags + FLAGS = flags.FLAGS # NOTE(vish): a user with one of these roles will be a superuser and @@ -99,6 +101,7 @@ class AuthBase(object): class User(AuthBase): """Object representing a user""" def __init__(self, id, name, access, secret, admin): + AuthBase.__init__(self) self.id = id self.name = name self.access = access @@ -159,6 +162,7 @@ class KeyPair(AuthBase): fingerprint is stored. The user's private key is not saved. """ def __init__(self, id, name, owner_id, public_key, fingerprint): + AuthBase.__init__(self) self.id = id self.name = name self.owner_id = owner_id @@ -176,6 +180,7 @@ class KeyPair(AuthBase): class Project(AuthBase): """Represents a Project returned from the datastore""" def __init__(self, id, name, project_manager_id, description, member_ids): + AuthBase.__init__(self) self.id = id self.name = name self.project_manager_id = project_manager_id @@ -234,7 +239,7 @@ class AuthManager(object): AuthManager also manages associated data related to Auth objects that need to be more accessible, such as vpn ips and ports. """ - _instance=None + _instance = None def __new__(cls, *args, **kwargs): """Returns the AuthManager singleton""" if not cls._instance: @@ -248,7 +253,7 @@ class AuthManager(object): reset the driver if it is not set or a new driver is specified. """ if driver or not getattr(self, 'driver', None): - self.driver = utils.import_class(driver or FLAGS.auth_driver) + self.driver = utils.import_class(driver or FLAGS.auth_driver) def authenticate(self, access, signature, params, verb='GET', server_string='127.0.0.1:8773', path='/', From 7a40f9bce0f90a9fcc7e9b7c8793d739f9e27bb4 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Mon, 9 Aug 2010 13:23:19 +0100 Subject: [PATCH 21/64] Move the xenapi top level directory under plugins, as suggested by Jay Pipes. --- xenapi/README | 2 - xenapi/etc/xapi.d/plugins/objectstore | 231 -------------------- xenapi/etc/xapi.d/plugins/pluginlib_nova.py | 216 ------------------ 3 files changed, 449 deletions(-) delete mode 100644 xenapi/README delete mode 100644 xenapi/etc/xapi.d/plugins/objectstore delete mode 100755 xenapi/etc/xapi.d/plugins/pluginlib_nova.py diff --git a/xenapi/README b/xenapi/README deleted file mode 100644 index 1fc67aa7..00000000 --- a/xenapi/README +++ /dev/null @@ -1,2 +0,0 @@ -This directory contains files that are required for the XenAPI support. They -should be installed in the XenServer / Xen Cloud Platform domain 0. diff --git a/xenapi/etc/xapi.d/plugins/objectstore b/xenapi/etc/xapi.d/plugins/objectstore deleted file mode 100644 index 271e7337..00000000 --- a/xenapi/etc/xapi.d/plugins/objectstore +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2010 Citrix Systems, Inc. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# -# XenAPI plugin for fetching images from nova-objectstore. -# - -import base64 -import errno -import hmac -import os -import os.path -import sha -import time -import urlparse - -import XenAPIPlugin - -from pluginlib_nova import * -configure_logging('objectstore') - - -KERNEL_DIR = '/boot/guest' - -DOWNLOAD_CHUNK_SIZE = 2 * 1024 * 1024 -SECTOR_SIZE = 512 -MBR_SIZE_SECTORS = 63 -MBR_SIZE_BYTES = MBR_SIZE_SECTORS * SECTOR_SIZE - - -def get_vdi(session, args): - src_url = exists(args, 'src_url') - username = exists(args, 'username') - password = exists(args, 'password') - add_partition = validate_bool(args, 'add_partition', 'false') - - (proto, netloc, url_path, _, _, _) = urlparse.urlparse(src_url) - - sr = find_sr(session) - if sr is None: - raise Exception('Cannot find SR to write VDI to') - - virtual_size = \ - get_content_length(proto, netloc, url_path, username, password) - if virtual_size < 0: - raise Exception('Cannot get VDI size') - - vdi_size = virtual_size - if add_partition: - # Make room for MBR. - vdi_size += MBR_SIZE_BYTES - - vdi = create_vdi(session, sr, src_url, vdi_size, False) - with_vdi_in_dom0(session, vdi, False, - lambda dev: get_vdi_(proto, netloc, url_path, - username, password, add_partition, - virtual_size, '/dev/%s' % dev)) - return session.xenapi.VDI.get_uuid(vdi) - - -def get_vdi_(proto, netloc, url_path, username, password, add_partition, - virtual_size, dest): - - if add_partition: - write_partition(virtual_size, dest) - - offset = add_partition and MBR_SIZE_BYTES or 0 - get(proto, netloc, url_path, username, password, dest, offset) - - -def write_partition(virtual_size, dest): - mbr_last = MBR_SIZE_SECTORS - 1 - primary_first = MBR_SIZE_SECTORS - primary_last = MBR_SIZE_SECTORS + (virtual_size / SECTOR_SIZE) - 1 - - logging.debug('Writing partition table %d %d to %s...', - primary_first, primary_last, dest) - - result = os.system('parted --script %s mklabel msdos' % dest) - if result != 0: - raise Exception('Failed to mklabel') - result = os.system('parted --script %s mkpart primary %ds %ds' % - (dest, primary_first, primary_last)) - if result != 0: - raise Exception('Failed to mkpart') - - logging.debug('Writing partition table %s done.', dest) - - -def find_sr(session): - host = get_this_host(session) - srs = session.xenapi.SR.get_all() - for sr in srs: - sr_rec = session.xenapi.SR.get_record(sr) - if not ('i18n-key' in sr_rec['other_config'] and - sr_rec['other_config']['i18n-key'] == 'local-storage'): - continue - for pbd in sr_rec['PBDs']: - pbd_rec = session.xenapi.PBD.get_record(pbd) - if pbd_rec['host'] == host: - return sr - return None - - -def get_kernel(session, args): - src_url = exists(args, 'src_url') - username = exists(args, 'username') - password = exists(args, 'password') - - (proto, netloc, url_path, _, _, _) = urlparse.urlparse(src_url) - - dest = os.path.join(KERNEL_DIR, url_path[1:]) - - # Paranoid check against people using ../ to do rude things. - if os.path.commonprefix([KERNEL_DIR, dest]) != KERNEL_DIR: - raise Exception('Illegal destination %s %s', (url_path, dest)) - - dirname = os.path.dirname(dest) - try: - os.makedirs(dirname) - except os.error, e: - if e.errno != errno.EEXIST: - raise - if not os.path.isdir(dirname): - raise Exception('Cannot make directory %s', dirname) - - try: - os.remove(dest) - except: - pass - - get(proto, netloc, url_path, username, password, dest, 0) - - return dest - - -def get_content_length(proto, netloc, url_path, username, password): - headers = make_headers('HEAD', url_path, username, password) - return with_http_connection( - proto, netloc, - lambda conn: get_content_length_(url_path, headers, conn)) - - -def get_content_length_(url_path, headers, conn): - conn.request('HEAD', url_path, None, headers) - response = conn.getresponse() - if response.status != 200: - raise Exception('%d %s' % (response.status, response.reason)) - - return long(response.getheader('Content-Length', -1)) - - -def get(proto, netloc, url_path, username, password, dest, offset): - headers = make_headers('GET', url_path, username, password) - download(proto, netloc, url_path, headers, dest, offset) - - -def make_headers(verb, url_path, username, password): - headers = {} - headers['Date'] = \ - time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) - headers['Authorization'] = \ - 'AWS %s:%s' % (username, - s3_authorization(verb, url_path, password, headers)) - return headers - - -def s3_authorization(verb, path, password, headers): - sha1 = hmac.new(password, digestmod=sha) - sha1.update(plaintext(verb, path, headers)) - return base64.encodestring(sha1.digest()).strip() - - -def plaintext(verb, path, headers): - return '%s\n\n\n%s\n%s' % (verb, - "\n".join([headers[h] for h in headers]), - path) - - -def download(proto, netloc, url_path, headers, dest, offset): - with_http_connection( - proto, netloc, - lambda conn: download_(url_path, dest, offset, headers, conn)) - - -def download_(url_path, dest, offset, headers, conn): - conn.request('GET', url_path, None, headers) - response = conn.getresponse() - if response.status != 200: - raise Exception('%d %s' % (response.status, response.reason)) - - length = response.getheader('Content-Length', -1) - - with_file( - dest, 'a', - lambda dest_file: download_all(response, length, dest_file, offset)) - - -def download_all(response, length, dest_file, offset): - dest_file.seek(offset) - i = 0 - while True: - buf = response.read(DOWNLOAD_CHUNK_SIZE) - if buf: - dest_file.write(buf) - else: - return - i += len(buf) - if length != -1 and i >= length: - return - - -if __name__ == '__main__': - XenAPIPlugin.dispatch({'get_vdi': get_vdi, - 'get_kernel': get_kernel}) diff --git a/xenapi/etc/xapi.d/plugins/pluginlib_nova.py b/xenapi/etc/xapi.d/plugins/pluginlib_nova.py deleted file mode 100755 index 2d323a01..00000000 --- a/xenapi/etc/xapi.d/plugins/pluginlib_nova.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright (c) 2010 Citrix Systems, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# -# Helper functions for the Nova xapi plugins. In time, this will merge -# with the pluginlib.py shipped with xapi, but for now, that file is not -# very stable, so it's easiest just to have a copy of all the functions -# that we need. -# - -import httplib -import logging -import logging.handlers -import re -import time - - -##### Logging setup - -def configure_logging(name): - log = logging.getLogger() - log.setLevel(logging.DEBUG) - sysh = logging.handlers.SysLogHandler('/dev/log') - sysh.setLevel(logging.DEBUG) - formatter = logging.Formatter('%s: %%(levelname)-8s %%(message)s' % name) - sysh.setFormatter(formatter) - log.addHandler(sysh) - - -##### Exceptions - -class PluginError(Exception): - """Base Exception class for all plugin errors.""" - def __init__(self, *args): - Exception.__init__(self, *args) - -class ArgumentError(PluginError): - """Raised when required arguments are missing, argument values are invalid, - or incompatible arguments are given. - """ - def __init__(self, *args): - PluginError.__init__(self, *args) - - -##### Helpers - -def ignore_failure(func, *args, **kwargs): - try: - return func(*args, **kwargs) - except XenAPI.Failure, e: - logging.error('Ignoring XenAPI.Failure %s', e) - return None - - -##### Argument validation - -ARGUMENT_PATTERN = re.compile(r'^[a-zA-Z0-9_:\.\-,]+$') - -def validate_exists(args, key, default=None): - """Validates that a string argument to a RPC method call is given, and - matches the shell-safe regex, with an optional default value in case it - does not exist. - - Returns the string. - """ - if key in args: - if len(args[key]) == 0: - raise ArgumentError('Argument %r value %r is too short.' % (key, args[key])) - if not ARGUMENT_PATTERN.match(args[key]): - raise ArgumentError('Argument %r value %r contains invalid characters.' % (key, args[key])) - if args[key][0] == '-': - raise ArgumentError('Argument %r value %r starts with a hyphen.' % (key, args[key])) - return args[key] - elif default is not None: - return default - else: - raise ArgumentError('Argument %s is required.' % key) - -def validate_bool(args, key, default=None): - """Validates that a string argument to a RPC method call is a boolean string, - with an optional default value in case it does not exist. - - Returns the python boolean value. - """ - value = validate_exists(args, key, default) - if value.lower() == 'true': - return True - elif value.lower() == 'false': - return False - else: - raise ArgumentError("Argument %s may not take value %r. Valid values are ['true', 'false']." % (key, value)) - -def exists(args, key): - """Validates that a freeform string argument to a RPC method call is given. - Returns the string. - """ - if key in args: - return args[key] - else: - raise ArgumentError('Argument %s is required.' % key) - -def optional(args, key): - """If the given key is in args, return the corresponding value, otherwise - return None""" - return key in args and args[key] or None - - -def get_this_host(session): - return session.xenapi.session.get_this_host(session.handle) - - -def get_domain_0(session): - this_host_ref = get_this_host(session) - expr = 'field "is_control_domain" = "true" and field "resident_on" = "%s"' % this_host_ref - return session.xenapi.VM.get_all_records_where(expr).keys()[0] - - -def create_vdi(session, sr_ref, name_label, virtual_size, read_only): - vdi_ref = session.xenapi.VDI.create( - { 'name_label': name_label, - 'name_description': '', - 'SR': sr_ref, - 'virtual_size': str(virtual_size), - 'type': 'User', - 'sharable': False, - 'read_only': read_only, - 'xenstore_data': {}, - 'other_config': {}, - 'sm_config': {}, - 'tags': [] }) - logging.debug('Created VDI %s (%s, %s, %s) on %s.', vdi_ref, name_label, - virtual_size, read_only, sr_ref) - return vdi_ref - - -def with_vdi_in_dom0(session, vdi, read_only, f): - dom0 = get_domain_0(session) - vbd_rec = {} - vbd_rec['VM'] = dom0 - vbd_rec['VDI'] = vdi - vbd_rec['userdevice'] = 'autodetect' - vbd_rec['bootable'] = False - vbd_rec['mode'] = read_only and 'RO' or 'RW' - vbd_rec['type'] = 'disk' - vbd_rec['unpluggable'] = True - vbd_rec['empty'] = False - vbd_rec['other_config'] = {} - vbd_rec['qos_algorithm_type'] = '' - vbd_rec['qos_algorithm_params'] = {} - vbd_rec['qos_supported_algorithms'] = [] - logging.debug('Creating VBD for VDI %s ... ', vdi) - vbd = session.xenapi.VBD.create(vbd_rec) - logging.debug('Creating VBD for VDI %s done.', vdi) - try: - logging.debug('Plugging VBD %s ... ', vbd) - session.xenapi.VBD.plug(vbd) - logging.debug('Plugging VBD %s done.', vbd) - return f(session.xenapi.VBD.get_device(vbd)) - finally: - logging.debug('Destroying VBD for VDI %s ... ', vdi) - vbd_unplug_with_retry(session, vbd) - ignore_failure(session.xenapi.VBD.destroy, vbd) - logging.debug('Destroying VBD for VDI %s done.', vdi) - - -def vbd_unplug_with_retry(session, vbd): - """Call VBD.unplug on the given VBD, with a retry if we get - DEVICE_DETACH_REJECTED. For reasons which I don't understand, we're - seeing the device still in use, even when all processes using the device - should be dead.""" - while True: - try: - session.xenapi.VBD.unplug(vbd) - logging.debug('VBD.unplug successful first time.') - return - except XenAPI.Failure, e: - if (len(e.details) > 0 and - e.details[0] == 'DEVICE_DETACH_REJECTED'): - logging.debug('VBD.unplug rejected: retrying...') - time.sleep(1) - elif (len(e.details) > 0 and - e.details[0] == 'DEVICE_ALREADY_DETACHED'): - logging.debug('VBD.unplug successful eventually.') - return - else: - logging.error('Ignoring XenAPI.Failure in VBD.unplug: %s', e) - return - - -def with_http_connection(proto, netloc, f): - conn = (proto == 'https' and - httplib.HTTPSConnection(netloc) or - httplib.HTTPConnection(netloc)) - try: - return f(conn) - finally: - conn.close() - - -def with_file(dest_path, mode, f): - dest = open(dest_path, mode) - try: - return f(dest) - finally: - dest.close() From cbb2486cad02e3bf4011ef483490a65880aebcfe Mon Sep 17 00:00:00 2001 From: Eric Day Date: Mon, 9 Aug 2010 07:31:41 -0700 Subject: [PATCH 22/64] Fixed docstring format per Jay's review. --- nova/endpoint/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index f6735a26..75b828e9 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -105,7 +105,7 @@ class CloudServerAPI(wsgi.Application): return json.dumps(value) def instance_details(self, inst): # pylint: disable-msg=R0201 - "Build the data structure to represent details for an instance." + """Build the data structure to represent details for an instance.""" return { "id": inst.get("instance_id", None), "imageId": inst.get("image_id", None), From 4d2757eaf877627a5c9f0a5c82ee0dc887ebe4b0 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Mon, 9 Aug 2010 10:46:33 -0400 Subject: [PATCH 23/64] pylint fixes for nova/server.py --- nova/server.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/server.py b/nova/server.py index 7a1901a2..96550f07 100644 --- a/nova/server.py +++ b/nova/server.py @@ -52,13 +52,8 @@ def stop(pidfile): """ # Get the pid from the pidfile try: - pf = file(pidfile,'r') - pid = int(pf.read().strip()) - pf.close() + pid = int(open(pidfile,'r').read().strip()) except IOError: - pid = None - - if not pid: message = "pidfile %s does not exist. Daemon not running?\n" sys.stderr.write(message % pidfile) return # not an error in a restart @@ -79,14 +74,15 @@ def stop(pidfile): def serve(name, main): + """Controller for server""" argv = FLAGS(sys.argv) if not FLAGS.pidfile: FLAGS.pidfile = '%s.pid' % name - logging.debug("Full set of FLAGS: \n\n\n" ) + logging.debug("Full set of FLAGS: \n\n\n") for flag in FLAGS: - logging.debug("%s : %s" % (flag, FLAGS.get(flag, None) )) + logging.debug("%s : %s", flag, FLAGS.get(flag, None)) action = 'start' if len(argv) > 1: @@ -102,7 +98,11 @@ def serve(name, main): else: print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) + daemonize(argv, name, main) + +def daemonize(args, name, main): + """Does the work of daemonizing the process""" logging.getLogger('amqplib').setLevel(logging.WARN) if FLAGS.daemonize: logger = logging.getLogger() @@ -115,7 +115,7 @@ def serve(name, main): else: if not FLAGS.logfile: FLAGS.logfile = '%s.log' % name - logfile = logging.handlers.FileHandler(FLAGS.logfile) + logfile = logging.FileHandler(FLAGS.logfile) logfile.setFormatter(formatter) logger.addHandler(logfile) stdin, stdout, stderr = None, None, None @@ -137,4 +137,4 @@ def serve(name, main): stdout=stdout, stderr=stderr ): - main(argv) + main(args) From 4e7d7f8418561c73e5fc6bff6b990c7961c82cf6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 9 Aug 2010 15:34:05 -0700 Subject: [PATCH 24/64] initial cleanup of tests for network --- nova/tests/network_unittest.py | 106 +++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 38 deletions(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 879ee02a..94d10200 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -54,6 +54,7 @@ class NetworkTestCase(test.TrialTestCase): self.projects.append(self.manager.create_project(name, 'netuser', name)) + vpn.NetworkData.create(self.projects[i].id) self.network = model.PublicNetworkController() self.service = service.VlanNetworkService() @@ -70,7 +71,7 @@ class NetworkTestCase(test.TrialTestCase): self.assertTrue(IPy.IP(address) in self.network.network) def test_allocate_deallocate_fixed_ip(self): - result = yield self.service.allocate_fixed_ip( + result = self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) address = result['private_dns_name'] mac = result['mac_address'] @@ -89,11 +90,11 @@ class NetworkTestCase(test.TrialTestCase): def test_range_allocation(self): hostname = "test-host" - result = yield self.service.allocate_fixed_ip( + result = self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) mac = result['mac_address'] address = result['private_dns_name'] - result = yield self.service.allocate_fixed_ip( + result = self.service.allocate_fixed_ip( self.user, self.projects[1].id) secondmac = result['mac_address'] secondaddress = result['private_dns_name'] @@ -123,21 +124,21 @@ class NetworkTestCase(test.TrialTestCase): self.assertEqual(False, is_in_project(secondaddress, self.projects[1].id)) def test_subnet_edge(self): - result = yield self.service.allocate_fixed_ip(self.user.id, + result = self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) firstaddress = result['private_dns_name'] hostname = "toomany-hosts" for i in range(1,5): project_id = self.projects[i].id - result = yield self.service.allocate_fixed_ip( + result = self.service.allocate_fixed_ip( self.user, project_id) mac = result['mac_address'] address = result['private_dns_name'] - result = yield self.service.allocate_fixed_ip( + result = self.service.allocate_fixed_ip( self.user, project_id) mac2 = result['mac_address'] address2 = result['private_dns_name'] - result = yield self.service.allocate_fixed_ip( + result = self.service.allocate_fixed_ip( self.user, project_id) mac3 = result['mac_address'] address3 = result['private_dns_name'] @@ -155,8 +156,7 @@ class NetworkTestCase(test.TrialTestCase): rv = self.service.deallocate_fixed_ip(firstaddress) self.dnsmasq.release_ip(mac, firstaddress, hostname, net.bridge_name) - def test_212_vpn_ip_and_port_looks_valid(self): - vpn.NetworkData.create(self.projects[0].id) + def test_vpn_ip_and_port_looks_valid(self): self.assert_(self.projects[0].vpn_ip) self.assert_(self.projects[0].vpn_port >= FLAGS.vpn_start_port) self.assert_(self.projects[0].vpn_port <= FLAGS.vpn_end_port) @@ -169,55 +169,85 @@ class NetworkTestCase(test.TrialTestCase): for network_datum in vpns: network_datum.destroy() - def test_release_before_deallocate(self): - pass + def test_ips_are_reused(self): + """Makes sure that ip addresses that are deallocated get reused""" - def test_deallocate_before_issued(self): - pass + result = self.service.allocate_fixed_ip( + self.user.id, self.projects[0].id) + mac = result['mac_address'] + address = result['private_dns_name'] - def test_too_many_addresses(self): - """ - Here, we test that a proper NoMoreAddresses exception is raised. + hostname = "reuse-host" + net = model.get_project_network(self.projects[0].id, "default") - However, the number of available IP addresses depends on the test + self.dnsmasq.issue_ip(mac, address, hostname, net.bridge_name) + rv = self.service.deallocate_fixed_ip(address) + self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name) + + result = self.service.allocate_fixed_ip( + self.user, self.projects[0].id) + secondmac = result['mac_address'] + secondaddress = result['private_dns_name'] + self.assertEqual(address, secondaddress) + rv = self.service.deallocate_fixed_ip(secondaddress) + self.dnsmasq.issue_ip(secondmac, + secondaddress, + hostname, + net.bridge_name) + self.dnsmasq.release_ip(secondmac, + secondaddress, + hostname, + net.bridge_name) + + def test_available_ips(self): + """Make sure the number of available ips for the network is correct + + The number of available IP addresses depends on the test environment's setup. Network size is set in test fixture's setUp method. - There are FLAGS.cnt_vpn_clients addresses reserved for VPN (NUM_RESERVED_VPN_IPS) - - And there are NUM_STATIC_IPS that are always reserved by Nova for the necessary - services (gateway, CloudPipe, etc) - - So we should get flags.network_size - (NUM_STATIC_IPS + - NUM_PREALLOCATED_IPS + - NUM_RESERVED_VPN_IPS) - usable addresses + There are ips reserved at the bottom and top of the range. + services (network, gateway, CloudPipe, broadcast) """ net = model.get_project_network(self.projects[0].id, "default") - - # Determine expected number of available IP addresses - num_static_ips = net.num_static_ips num_preallocated_ips = len(net.hosts.keys()) - num_reserved_vpn_ips = flags.FLAGS.cnt_vpn_clients - num_available_ips = flags.FLAGS.network_size - (num_static_ips + + num_available_ips = flags.FLAGS.network_size - (net.num_bottom_reserved_ips + num_preallocated_ips + - num_reserved_vpn_ips) + net.num_top_reserved_ips) + self.assertEqual(num_available_ips, len(list(net.available))) + + def test_too_many_addresses(self): + """Test for a NoMoreAddresses exception when all fixed ips are used. + """ + net = model.get_project_network(self.projects[0].id, "default") hostname = "toomany-hosts" macs = {} addresses = {} - for i in range(0, (num_available_ips - 1)): - result = yield self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) + # Number of availaible ips is len of the available list + num_available_ips = len(list(net.available)) + for i in range(num_available_ips): + result = self.service.allocate_fixed_ip(self.user.id, + self.projects[0].id) macs[i] = result['mac_address'] addresses[i] = result['private_dns_name'] - self.dnsmasq.issue_ip(macs[i], addresses[i], hostname, net.bridge_name) + self.dnsmasq.issue_ip(macs[i], + addresses[i], + hostname, + net.bridge_name) - self.assertFailure(self.service.allocate_fixed_ip(self.user.id, self.projects[0].id), NoMoreAddresses) + self.assertEqual(len(list(net.available)), 0) + self.assertRaises(NoMoreAddresses, self.service.allocate_fixed_ip, + self.user.id, self.projects[0].id) - for i in range(0, (num_available_ips - 1)): + for i in range(len(addresses)): rv = self.service.deallocate_fixed_ip(addresses[i]) - self.dnsmasq.release_ip(macs[i], addresses[i], hostname, net.bridge_name) + self.dnsmasq.release_ip(macs[i], + addresses[i], + hostname, + net.bridge_name) + self.assertEqual(len(list(net.available)), num_available_ips) def is_in_project(address, project_id): return address in model.get_project_network(project_id).list_addresses() From 9cc5c5183a6d6b41310dc6864449094265a4662f Mon Sep 17 00:00:00 2001 From: Joe Heck Date: Mon, 9 Aug 2010 17:53:10 -0700 Subject: [PATCH 25/64] fixing - removing unused imports per Eric & Jay review --- nova/auth/manager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index e5efbca2..6d71a7ad 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -35,10 +35,6 @@ from nova import utils from nova.auth import signer from nova.network import vpn -#unused imports -#from nova import datastore -#from nova.auth import ldapdriver # for flags -#from nova import objectstore # for flags FLAGS = flags.FLAGS From 7b9ca222d12bdb96d36b7d913d2331c0df8ddab0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 10:25:52 -0700 Subject: [PATCH 26/64] rename create_zip to zipfile so lazy match works --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 2dd569df..6af09292 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -203,7 +203,7 @@ class ProjectCommands(object): arguments: project user""" self.manager.remove_from_project(user, project) - def create_zip(self, project_id, user_id, filename='nova.zip'): + def zipfile(self, project_id, user_id, filename='nova.zip'): """Exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" zip_file = self.manager.get_credentials(project_id, user_id) From b92c66bd8c88511c61ea2df16e3c4ba2f3343c6d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 11:48:14 -0700 Subject: [PATCH 27/64] pep8 cleanup --- nova/tests/network_unittest.py | 50 +++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 94d10200..9aa39e51 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -31,6 +31,7 @@ from nova.network.exception import NoMoreAddresses FLAGS = flags.FLAGS + class NetworkTestCase(test.TrialTestCase): def setUp(self): super(NetworkTestCase, self).setUp() @@ -66,12 +67,14 @@ class NetworkTestCase(test.TrialTestCase): def test_public_network_allocation(self): pubnet = IPy.IP(flags.FLAGS.public_range) - address = self.network.allocate_ip(self.user.id, self.projects[0].id, "public") + address = self.network.allocate_ip(self.user.id, + self.projects[0].id, + "public") self.assertTrue(IPy.IP(address) in pubnet) self.assertTrue(IPy.IP(address) in self.network.network) def test_allocate_deallocate_fixed_ip(self): - result = self.service.allocate_fixed_ip( + result = self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) address = result['private_dns_name'] mac = result['mac_address'] @@ -103,7 +106,8 @@ class NetworkTestCase(test.TrialTestCase): secondnet = model.get_project_network(self.projects[1].id, "default") self.assertEqual(True, is_in_project(address, self.projects[0].id)) - self.assertEqual(True, is_in_project(secondaddress, self.projects[1].id)) + self.assertEqual(True, is_in_project(secondaddress, + self.projects[1].id)) self.assertEqual(False, is_in_project(address, self.projects[1].id)) # Addresses are allocated before they're issued @@ -116,19 +120,21 @@ class NetworkTestCase(test.TrialTestCase): self.assertEqual(False, is_in_project(address, self.projects[0].id)) # First address release shouldn't affect the second - self.assertEqual(True, is_in_project(secondaddress, self.projects[1].id)) + self.assertEqual(True, is_in_project(secondaddress, + self.projects[1].id)) rv = self.service.deallocate_fixed_ip(secondaddress) self.dnsmasq.release_ip(secondmac, secondaddress, hostname, secondnet.bridge_name) - self.assertEqual(False, is_in_project(secondaddress, self.projects[1].id)) + self.assertEqual(False, is_in_project(secondaddress, + self.projects[1].id)) def test_subnet_edge(self): result = self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) firstaddress = result['private_dns_name'] hostname = "toomany-hosts" - for i in range(1,5): + for i in range(1, 5): project_id = self.projects[i].id result = self.service.allocate_fixed_ip( self.user, project_id) @@ -142,9 +148,12 @@ class NetworkTestCase(test.TrialTestCase): self.user, project_id) mac3 = result['mac_address'] address3 = result['private_dns_name'] - self.assertEqual(False, is_in_project(address, self.projects[0].id)) - self.assertEqual(False, is_in_project(address2, self.projects[0].id)) - self.assertEqual(False, is_in_project(address3, self.projects[0].id)) + self.assertEqual(False, is_in_project(address, + self.projects[0].id)) + self.assertEqual(False, is_in_project(address2, + self.projects[0].id)) + self.assertEqual(False, is_in_project(address3, + self.projects[0].id)) rv = self.service.deallocate_fixed_ip(address) rv = self.service.deallocate_fixed_ip(address2) rv = self.service.deallocate_fixed_ip(address3) @@ -212,9 +221,10 @@ class NetworkTestCase(test.TrialTestCase): """ net = model.get_project_network(self.projects[0].id, "default") num_preallocated_ips = len(net.hosts.keys()) - num_available_ips = flags.FLAGS.network_size - (net.num_bottom_reserved_ips + - num_preallocated_ips + - net.num_top_reserved_ips) + net_size = flags.FLAGS.network_size + num_available_ips = net_size - (net.num_bottom_reserved_ips + + num_preallocated_ips + + net.num_top_reserved_ips) self.assertEqual(num_available_ips, len(list(net.available))) def test_too_many_addresses(self): @@ -249,25 +259,22 @@ class NetworkTestCase(test.TrialTestCase): net.bridge_name) self.assertEqual(len(list(net.available)), num_available_ips) + def is_in_project(address, project_id): return address in model.get_project_network(project_id).list_addresses() -def _get_project_addresses(project_id): - project_addresses = [] - for addr in model.get_project_network(project_id).list_addresses(): - project_addresses.append(addr) - return project_addresses def binpath(script): return os.path.abspath(os.path.join(__file__, "../../../bin", script)) + class FakeDNSMasq(object): def issue_ip(self, mac, ip, hostname, interface): cmd = "%s add %s %s %s" % (binpath('nova-dhcpbridge'), mac, ip, hostname) env = {'DNSMASQ_INTERFACE': interface, - 'TESTING' : '1', - 'FLAGFILE' : FLAGS.dhcpbridge_flagfile} + 'TESTING': '1', + 'FLAGFILE': FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("ISSUE_IP: %s, %s " % (out, err)) @@ -275,8 +282,7 @@ class FakeDNSMasq(object): cmd = "%s del %s %s %s" % (binpath('nova-dhcpbridge'), mac, ip, hostname) env = {'DNSMASQ_INTERFACE': interface, - 'TESTING' : '1', - 'FLAGFILE' : FLAGS.dhcpbridge_flagfile} + 'TESTING': '1', + 'FLAGFILE': FLAGS.dhcpbridge_flagfile} (out, err) = utils.execute(cmd, addl_env=env) logging.debug("RELEASE_IP: %s, %s " % (out, err)) - From 42924d3bae051f97990a0cea0a0d4de299660e9c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 12:27:06 -0700 Subject: [PATCH 28/64] pylint cleanup --- bin/nova-dhcpbridge | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 7789dac9..6a9115fc 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -70,8 +70,9 @@ def init_leases(interface): net = model.get_network_by_interface(interface) res = "" for host_name in net.hosts: - res += "%s\n" % linux_net.hostDHCP(net, host_name, - net.hosts[host_name]) + res += "%s\n" % linux_net.host_dhcp(net, + host_name, + net.hosts[host_name]) return res From cad3517f2494e81de8795ffe1048ad5789d3289d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 12:46:40 -0700 Subject: [PATCH 29/64] Further pylint cleanup --- nova/endpoint/cloud.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index ad9188ff..02969c8e 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -103,7 +103,7 @@ class CloudController(object): result = {} for instance in self.instdir.all: if instance['project_id'] == project_id: - line = '%s slots=%d' % (instance['private_dns_name'], + line = '%s slots=%d' % (instance['private_dns_name'], INSTANCE_TYPES[instance['instance_type']]['vcpus']) if instance['key_name'] in result: result[instance['key_name']].append(line) @@ -423,7 +423,7 @@ class CloudController(object): i['key_name'] = instance.get('key_name', None) if context.user.is_admin(): i['key_name'] = '%s (%s, %s)' % (i['key_name'], - instance.get('project_id', None), + instance.get('project_id', None), instance.get('node_name', '')) i['product_codes_set'] = self._convert_to_set( instance.get('product_codes', None), 'product_code') @@ -560,15 +560,15 @@ class CloudController(object): # TODO: Get the real security group of launch in here security_group = "default" for num in range(int(kwargs['max_count'])): - vpn = False + is_vpn = False if image_id == FLAGS.vpn_image_id: - vpn = True + is_vpn = True allocate_result = yield rpc.call(network_topic, {"method": "allocate_fixed_ip", "args": {"user_id": context.user.id, "project_id": context.project.id, "security_group": security_group, - "vpn": vpn}}) + "is_vpn": is_vpn}}) allocate_data = allocate_result['result'] inst = self.instdir.new() inst['image_id'] = image_id From cfc9c3dac12737306a1e1f26e9b4922755160169 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 10 Aug 2010 18:17:44 -0400 Subject: [PATCH 30/64] work on a router that works with wsgi and non-wsgi routing --- nova/endpoint/rackspace.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace.py index 75b828e9..b4e6cd82 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace.py @@ -45,18 +45,20 @@ class API(wsgi.Middleware): def __init__(self): super(API, self).__init__(Router(webob.exc.HTTPNotFound())) - def __call__(self, environ, start_response): + @webob.dec.wsgify + def __call__(self, req): + return self.application context = {} - if "HTTP_X_AUTH_TOKEN" in environ: + if "HTTP_X_AUTH_TOKEN" in req.environ: context['user'] = manager.AuthManager().get_user_from_access_key( - environ['HTTP_X_AUTH_TOKEN']) + req.environ['HTTP_X_AUTH_TOKEN']) if context['user']: context['project'] = manager.AuthManager().get_project( context['user'].name) if "user" not in context: - return webob.exc.HTTPForbidden()(environ, start_response) + return webob.exc.HTTPForbidden() environ['nova.context'] = context - return self.application(environ, start_response) + return self.application class Router(wsgi.Router): @@ -64,13 +66,14 @@ class Router(wsgi.Router): def _build_map(self): """Build routing map for authentication and cloud.""" - self._connect("/v1.0", controller=AuthenticationAPI()) - cloud = CloudServerAPI() - self._connect("/servers", controller=cloud.launch_server, - conditions={"method": ["POST"]}) - self._connect("/servers/{server_id}", controller=cloud.delete_server, - conditions={'method': ["DELETE"]}) - self._connect("/servers", controller=cloud) + self.map.resource("server", "servers", controller=CloudServerAPI()) + #self._connect("/v1.0", controller=AuthenticationAPI()) + #cloud = CloudServerAPI() + #self._connect("/servers", controller=cloud.launch_server, + # conditions={"method": ["POST"]}) + #self._connect("/servers/{server_id}", controller=cloud.delete_server, + # conditions={'method': ["DELETE"]}) + #self._connect("/servers", controller=cloud) class AuthenticationAPI(wsgi.Application): From 865305272c3d7e2bda0bba1da69319015e27d871 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 15:45:24 -0700 Subject: [PATCH 31/64] lots more pylint fixes --- nova/tests/network_unittest.py | 145 ++++++++++++++++----------------- 1 file changed, 70 insertions(+), 75 deletions(-) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 9aa39e51..5671a888 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -15,7 +15,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +""" +Unit Tests for network code +""" import IPy import os import logging @@ -33,7 +35,8 @@ FLAGS = flags.FLAGS class NetworkTestCase(test.TrialTestCase): - def setUp(self): + """Test cases for network code""" + def setUp(self): # pylint: disable=C0103 super(NetworkTestCase, self).setUp() # NOTE(vish): if you change these flags, make sure to change the # flags in the corresponding section in nova-dhcpbridge @@ -44,7 +47,6 @@ class NetworkTestCase(test.TrialTestCase): network_size=32) logging.getLogger().setLevel(logging.DEBUG) self.manager = manager.AuthManager() - self.dnsmasq = FakeDNSMasq() self.user = self.manager.create_user('netuser', 'netuser', 'netuser') self.projects = [] self.projects.append(self.manager.create_project('netuser', @@ -56,49 +58,48 @@ class NetworkTestCase(test.TrialTestCase): 'netuser', name)) vpn.NetworkData.create(self.projects[i].id) - self.network = model.PublicNetworkController() self.service = service.VlanNetworkService() - def tearDown(self): + def tearDown(self): # pylint: disable=C0103 super(NetworkTestCase, self).tearDown() for project in self.projects: self.manager.delete_project(project) self.manager.delete_user(self.user) def test_public_network_allocation(self): + """Makes sure that we can allocaate a public ip""" pubnet = IPy.IP(flags.FLAGS.public_range) - address = self.network.allocate_ip(self.user.id, - self.projects[0].id, - "public") + address = self.service.allocate_elastic_ip(self.user.id, + self.projects[0].id) self.assertTrue(IPy.IP(address) in pubnet) - self.assertTrue(IPy.IP(address) in self.network.network) def test_allocate_deallocate_fixed_ip(self): + """Makes sure that we can allocate and deallocate a fixed ip""" result = self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) address = result['private_dns_name'] mac = result['mac_address'] - logging.debug("Was allocated %s" % (address)) net = model.get_project_network(self.projects[0].id, "default") self.assertEqual(True, is_in_project(address, self.projects[0].id)) hostname = "test-host" - self.dnsmasq.issue_ip(mac, address, hostname, net.bridge_name) - rv = self.service.deallocate_fixed_ip(address) + issue_ip(mac, address, hostname, net.bridge_name) + self.service.deallocate_fixed_ip(address) # Doesn't go away until it's dhcp released self.assertEqual(True, is_in_project(address, self.projects[0].id)) - self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name) + release_ip(mac, address, hostname, net.bridge_name) self.assertEqual(False, is_in_project(address, self.projects[0].id)) - def test_range_allocation(self): - hostname = "test-host" - result = self.service.allocate_fixed_ip( - self.user.id, self.projects[0].id) + def test_side_effects(self): + """Ensures allocating and releasing has no side effects""" + hostname = "side-effect-host" + result = self.service.allocate_fixed_ip(self.user.id, + self.projects[0].id) mac = result['mac_address'] address = result['private_dns_name'] - result = self.service.allocate_fixed_ip( - self.user, self.projects[1].id) + result = self.service.allocate_fixed_ip(self.user, + self.projects[1].id) secondmac = result['mac_address'] secondaddress = result['private_dns_name'] @@ -111,25 +112,24 @@ class NetworkTestCase(test.TrialTestCase): self.assertEqual(False, is_in_project(address, self.projects[1].id)) # Addresses are allocated before they're issued - self.dnsmasq.issue_ip(mac, address, hostname, net.bridge_name) - self.dnsmasq.issue_ip(secondmac, secondaddress, - hostname, secondnet.bridge_name) + issue_ip(mac, address, hostname, net.bridge_name) + issue_ip(secondmac, secondaddress, hostname, secondnet.bridge_name) - rv = self.service.deallocate_fixed_ip(address) - self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name) + self.service.deallocate_fixed_ip(address) + release_ip(mac, address, hostname, net.bridge_name) self.assertEqual(False, is_in_project(address, self.projects[0].id)) # First address release shouldn't affect the second self.assertEqual(True, is_in_project(secondaddress, self.projects[1].id)) - rv = self.service.deallocate_fixed_ip(secondaddress) - self.dnsmasq.release_ip(secondmac, secondaddress, - hostname, secondnet.bridge_name) + self.service.deallocate_fixed_ip(secondaddress) + release_ip(secondmac, secondaddress, hostname, secondnet.bridge_name) self.assertEqual(False, is_in_project(secondaddress, self.projects[1].id)) def test_subnet_edge(self): + """Makes sure that private ips don't overlap""" result = self.service.allocate_fixed_ip(self.user.id, self.projects[0].id) firstaddress = result['private_dns_name'] @@ -148,29 +148,34 @@ class NetworkTestCase(test.TrialTestCase): self.user, project_id) mac3 = result['mac_address'] address3 = result['private_dns_name'] + net = model.get_project_network(project_id, "default") + issue_ip(mac, address, hostname, net.bridge_name) + issue_ip(mac2, address2, hostname, net.bridge_name) + issue_ip(mac3, address3, hostname, net.bridge_name) self.assertEqual(False, is_in_project(address, self.projects[0].id)) self.assertEqual(False, is_in_project(address2, self.projects[0].id)) self.assertEqual(False, is_in_project(address3, self.projects[0].id)) - rv = self.service.deallocate_fixed_ip(address) - rv = self.service.deallocate_fixed_ip(address2) - rv = self.service.deallocate_fixed_ip(address3) - net = model.get_project_network(project_id, "default") - self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name) - self.dnsmasq.release_ip(mac2, address2, hostname, net.bridge_name) - self.dnsmasq.release_ip(mac3, address3, hostname, net.bridge_name) + self.service.deallocate_fixed_ip(address) + self.service.deallocate_fixed_ip(address2) + self.service.deallocate_fixed_ip(address3) + release_ip(mac, address, hostname, net.bridge_name) + release_ip(mac2, address2, hostname, net.bridge_name) + release_ip(mac3, address3, hostname, net.bridge_name) net = model.get_project_network(self.projects[0].id, "default") - rv = self.service.deallocate_fixed_ip(firstaddress) - self.dnsmasq.release_ip(mac, firstaddress, hostname, net.bridge_name) + self.service.deallocate_fixed_ip(firstaddress) + release_ip(mac, firstaddress, hostname, net.bridge_name) def test_vpn_ip_and_port_looks_valid(self): + """Ensure the vpn ip and port are reasonable""" self.assert_(self.projects[0].vpn_ip) self.assert_(self.projects[0].vpn_port >= FLAGS.vpn_start_port) self.assert_(self.projects[0].vpn_port <= FLAGS.vpn_end_port) def test_too_many_vpns(self): + """Ensure error is raised if we run out of vpn ports""" vpns = [] for i in xrange(vpn.NetworkData.num_ports_for_ip(FLAGS.vpn_ip)): vpns.append(vpn.NetworkData.create("vpnuser%s" % i)) @@ -180,7 +185,6 @@ class NetworkTestCase(test.TrialTestCase): def test_ips_are_reused(self): """Makes sure that ip addresses that are deallocated get reused""" - result = self.service.allocate_fixed_ip( self.user.id, self.projects[0].id) mac = result['mac_address'] @@ -189,24 +193,18 @@ class NetworkTestCase(test.TrialTestCase): hostname = "reuse-host" net = model.get_project_network(self.projects[0].id, "default") - self.dnsmasq.issue_ip(mac, address, hostname, net.bridge_name) - rv = self.service.deallocate_fixed_ip(address) - self.dnsmasq.release_ip(mac, address, hostname, net.bridge_name) + issue_ip(mac, address, hostname, net.bridge_name) + self.service.deallocate_fixed_ip(address) + release_ip(mac, address, hostname, net.bridge_name) result = self.service.allocate_fixed_ip( self.user, self.projects[0].id) secondmac = result['mac_address'] secondaddress = result['private_dns_name'] self.assertEqual(address, secondaddress) - rv = self.service.deallocate_fixed_ip(secondaddress) - self.dnsmasq.issue_ip(secondmac, - secondaddress, - hostname, - net.bridge_name) - self.dnsmasq.release_ip(secondmac, - secondaddress, - hostname, - net.bridge_name) + self.service.deallocate_fixed_ip(secondaddress) + issue_ip(secondmac, secondaddress, hostname, net.bridge_name) + release_ip(secondmac, secondaddress, hostname, net.bridge_name) def test_available_ips(self): """Make sure the number of available ips for the network is correct @@ -242,47 +240,44 @@ class NetworkTestCase(test.TrialTestCase): self.projects[0].id) macs[i] = result['mac_address'] addresses[i] = result['private_dns_name'] - self.dnsmasq.issue_ip(macs[i], - addresses[i], - hostname, - net.bridge_name) + issue_ip(macs[i], addresses[i], hostname, net.bridge_name) self.assertEqual(len(list(net.available)), 0) self.assertRaises(NoMoreAddresses, self.service.allocate_fixed_ip, self.user.id, self.projects[0].id) for i in range(len(addresses)): - rv = self.service.deallocate_fixed_ip(addresses[i]) - self.dnsmasq.release_ip(macs[i], - addresses[i], - hostname, - net.bridge_name) + self.service.deallocate_fixed_ip(addresses[i]) + release_ip(macs[i], addresses[i], hostname, net.bridge_name) self.assertEqual(len(list(net.available)), num_available_ips) def is_in_project(address, project_id): + """Returns true if address is in specified project""" return address in model.get_project_network(project_id).list_addresses() def binpath(script): + """Returns the absolute path to a script in bin""" return os.path.abspath(os.path.join(__file__, "../../../bin", script)) -class FakeDNSMasq(object): - def issue_ip(self, mac, ip, hostname, interface): - cmd = "%s add %s %s %s" % (binpath('nova-dhcpbridge'), - mac, ip, hostname) - env = {'DNSMASQ_INTERFACE': interface, - 'TESTING': '1', - 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) - logging.debug("ISSUE_IP: %s, %s " % (out, err)) +def issue_ip(mac, private_ip, hostname, interface): + """Run add command on dhcpbridge""" + cmd = "%s add %s %s %s" % (binpath('nova-dhcpbridge'), + mac, private_ip, hostname) + env = {'DNSMASQ_INTERFACE': interface, + 'TESTING': '1', + 'FLAGFILE': FLAGS.dhcpbridge_flagfile} + (out, err) = utils.execute(cmd, addl_env=env) + logging.debug("ISSUE_IP: %s, %s ", out, err) - def release_ip(self, mac, ip, hostname, interface): - cmd = "%s del %s %s %s" % (binpath('nova-dhcpbridge'), - mac, ip, hostname) - env = {'DNSMASQ_INTERFACE': interface, - 'TESTING': '1', - 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) - logging.debug("RELEASE_IP: %s, %s " % (out, err)) +def release_ip(mac, private_ip, hostname, interface): + """Run del command on dhcpbridge""" + cmd = "%s del %s %s %s" % (binpath('nova-dhcpbridge'), + mac, private_ip, hostname) + env = {'DNSMASQ_INTERFACE': interface, + 'TESTING': '1', + 'FLAGFILE': FLAGS.dhcpbridge_flagfile} + (out, err) = utils.execute(cmd, addl_env=env) + logging.debug("RELEASE_IP: %s, %s ", out, err) From d32ce6a65b662123f97acb303eba991eea1351c7 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 17:34:20 -0700 Subject: [PATCH 32/64] Adds get_roles commands to manager and driver classes --- nova/auth/ldapdriver.py | 34 +++++++++++++++++++++++++++------- nova/auth/manager.py | 18 ++++++++++++++++++ nova/tests/auth_unittest.py | 18 +++++++++++++++++- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index ec739e13..aaaf8553 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -181,7 +181,7 @@ class LdapDriver(object): if member_uids != None: for member_uid in member_uids: if not self.__user_exists(member_uid): - raise exception.NotFound("Project can't be created " + raise exception.NotFound("Project can't be created " "because user %s doesn't exist" % member_uid) members.append(self.__uid_to_dn(member_uid)) # always add the manager as a member because members is required @@ -236,6 +236,26 @@ class LdapDriver(object): role_dn = self.__role_to_dn(role, project_id) return self.__remove_from_group(uid, role_dn) + def get_user_roles(self, uid, project_id=None): + """Retrieve list of roles for user (or user and project)""" + if project_id is None: + # NOTE(vish): This is unneccesarily slow, but since we can't + # guarantee that the global roles are located + # together in the ldap tree, we're doing this version. + roles = [] + for role in FLAGS.allowed_roles: + role_dn = self.__role_to_dn(role) + if self.__is_in_group(uid, role_dn): + roles.append(role) + return roles + else: + project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) + roles = self.__find_objects(project_dn, + '(&(&(objectclass=groupOfNames)' + '(!(objectclass=novaProject)))' + '(member=%s))' % self.__uid_to_dn(uid)) + return [role['cn'][0] for role in roles] + def delete_user(self, uid): """Delete a user""" if not self.__user_exists(uid): @@ -253,24 +273,24 @@ class LdapDriver(object): self.conn.delete_s('cn=%s,uid=%s,%s' % (key_name, uid, FLAGS.ldap_user_subtree)) - def delete_project(self, name): + def delete_project(self, project_id): """Delete a project""" - project_dn = 'cn=%s,%s' % (name, FLAGS.ldap_project_subtree) + project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) self.__delete_roles(project_dn) self.__delete_group(project_dn) - def __user_exists(self, name): + def __user_exists(self, uid): """Check if user exists""" - return self.get_user(name) != None + return self.get_user(uid) != None def __key_pair_exists(self, uid, key_name): """Check if key pair exists""" return self.get_user(uid) != None return self.get_key_pair(uid, key_name) != None - def __project_exists(self, name): + def __project_exists(self, project_id): """Check if project exists""" - return self.get_project(name) != None + return self.get_project(project_id) != None def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 6d71a7ad..8195182f 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -38,6 +38,10 @@ from nova.network import vpn FLAGS = flags.FLAGS +flags.DEFINE_list('allowed_roles', + ['cloudadmin', 'itsec', 'sysadmin', 'netadmin', 'developer'], + 'Allowed roles for project') + # NOTE(vish): a user with one of these roles will be a superuser and # have access to all api commands flags.DEFINE_list('superuser_roles', ['cloudadmin'], @@ -455,6 +459,20 @@ class AuthManager(object): with self.driver() as drv: drv.remove_role(User.safe_id(user), role, Project.safe_id(project)) + def get_roles(self): + """Get list of allowed roles""" + return FLAGS.allowed_roles + + def get_user_roles(self, user, project=None): + """Get user global or per-project roles""" + roles = [] + with self.driver() as drv: + roles = drv.get_user_roles(User.safe_id(user), + Project.safe_id(project)) + if project is not None and self.is_project_manager(user, project): + roles.append('projectmanager') + return roles + def get_project(self, pid): """Get project object by id""" with self.driver() as drv: diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index f7e0625a..2d99c8e3 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -179,7 +179,23 @@ class AuthTestCase(test.BaseTestCase): project.add_role('test1', 'sysadmin') self.assertTrue(project.has_role('test1', 'sysadmin')) - def test_211_can_remove_project_role(self): + def test_211_can_list_project_roles(self): + project = self.manager.get_project('testproj') + user = self.manager.get_user('test1') + self.manager.add_role(user, 'netadmin', project) + roles = self.manager.get_user_roles(user) + self.assertTrue('sysadmin' in roles) + self.assertFalse('netadmin' in roles) + self.assertFalse('projectmanager' in roles) + project_roles = self.manager.get_user_roles(user, project) + self.assertTrue('sysadmin' in project_roles) + self.assertTrue('netadmin' in project_roles) + self.assertTrue('projectmanager' in project_roles) + # has role should be false because global role is missing + self.assertFalse(self.manager.has_role(user, 'netadmin', project)) + + + def test_212_can_remove_project_role(self): project = self.manager.get_project('testproj') self.assertTrue(project.has_role('test1', 'sysadmin')) project.remove_role('test1', 'sysadmin') From bf051b93aa74229fc33ad1ceff0c3c23556dda43 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 17:42:58 -0700 Subject: [PATCH 33/64] Throw exceptions for illegal roles on role add --- nova/auth/manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 8195182f..e338dfc8 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -436,6 +436,10 @@ class AuthManager(object): @type project: Project or project_id @param project: Project in which to add local role. """ + if role not in FLAGS.allowed_roles: + raise exception.NotFound("The %s role can not be found" % role) + if project is not None and role in FLAGS.global_roles: + raise exception.NotFound("The %s role is global only" % role) with self.driver() as drv: drv.add_role(User.safe_id(user), role, Project.safe_id(project)) From ac57eb423a280cec528be5cb44bd3415199336ee Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 18:04:23 -0700 Subject: [PATCH 34/64] change get_roles to have a flag for project_roles or not. Don't show 'projectmanager' in list of roles --- nova/auth/manager.py | 15 +++++++-------- nova/tests/auth_unittest.py | 2 -- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index e338dfc8..064fd78b 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -463,19 +463,18 @@ class AuthManager(object): with self.driver() as drv: drv.remove_role(User.safe_id(user), role, Project.safe_id(project)) - def get_roles(self): + def get_roles(self, project_roles=True): """Get list of allowed roles""" - return FLAGS.allowed_roles + if project_roles: + return list(set(FLAGS.allowed_roles) - set(FLAGS.global_roles)) + else: + return FLAGS.allowed_roles def get_user_roles(self, user, project=None): """Get user global or per-project roles""" - roles = [] with self.driver() as drv: - roles = drv.get_user_roles(User.safe_id(user), - Project.safe_id(project)) - if project is not None and self.is_project_manager(user, project): - roles.append('projectmanager') - return roles + return drv.get_user_roles(User.safe_id(user), + Project.safe_id(project)) def get_project(self, pid): """Get project object by id""" diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 2d99c8e3..0b404bfd 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -186,11 +186,9 @@ class AuthTestCase(test.BaseTestCase): roles = self.manager.get_user_roles(user) self.assertTrue('sysadmin' in roles) self.assertFalse('netadmin' in roles) - self.assertFalse('projectmanager' in roles) project_roles = self.manager.get_user_roles(user, project) self.assertTrue('sysadmin' in project_roles) self.assertTrue('netadmin' in project_roles) - self.assertTrue('projectmanager' in project_roles) # has role should be false because global role is missing self.assertFalse(self.manager.has_role(user, 'netadmin', project)) From ec5c25ea86850ad3ab16d0d3c8f859aee52c56b4 Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Tue, 10 Aug 2010 18:46:49 -0700 Subject: [PATCH 35/64] Wired up admin api for user roles --- nova/adminclient.py | 36 +++++++++++++++++++++++++++++++++++- nova/endpoint/admin.py | 13 +++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index 25d5e71c..5aa8ff9c 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -57,6 +57,28 @@ class UserInfo(object): elif name == 'secretkey': self.secretkey = str(value) +class UserRole(object): + """ + Information about a Nova user's role, as parsed through SAX. + Fields include: + role + """ + def __init__(self, connection=None): + self.connection = connection + self.role = None + + def __repr__(self): + return 'UserRole:%s' % self.role + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'role': + self.role = value + else: + setattr(self, name, str(value)) + class ProjectInfo(object): """ Information about a Nova project, as parsed through SAX @@ -114,7 +136,6 @@ class ProjectMember(object): else: setattr(self, name, str(value)) - class HostInfo(object): """ Information about a Nova Host, as parsed through SAX: @@ -196,6 +217,19 @@ class NovaAdminClient(object): """ deletes a user """ return self.apiconn.get_object('DeregisterUser', {'Name': username}, UserInfo) + def get_user_roles(self, user, project=None): + """ + Returns a list of roles for the given user. + Omitting project will return any global roles that the user has. + Specifying project will return only project specific roles. + """ + params = {'User':user} + if project: + params['Project'] = project + return self.apiconn.get_list('DescribeUserRoles', + params, + [('item', UserRole)]) + def add_user_role(self, user, role, project=None): """ Add a role to a user either globally or for a specific project. diff --git a/nova/endpoint/admin.py b/nova/endpoint/admin.py index c4b8c05c..a3114c0a 100644 --- a/nova/endpoint/admin.py +++ b/nova/endpoint/admin.py @@ -102,6 +102,19 @@ class AdminController(object): return True + @admin_only + def describe_roles(self, context, project_roles=True, **kwargs): + """Returns a list of allowed roles.""" + return manager.AuthManager().get_roles(project_roles) + + @admin_only + def describe_user_roles(self, context, user, project=None, **kwargs): + """Returns a list of roles for the given user. + Omitting project will return any global roles that the user has. + Specifying project will return only project specific roles. + """ + return manager.AuthManager().get_user_roles(user, project=project) + @admin_only def modify_user_role(self, context, user, role, project=None, operation='add', **kwargs): From 1c8a1736549212885a9ab08a7ee019e7563cd90e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 19:00:35 -0700 Subject: [PATCH 36/64] fix spacing issue in ldapdriver --- nova/auth/ldapdriver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index aaaf8553..453fa196 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -181,8 +181,9 @@ class LdapDriver(object): if member_uids != None: for member_uid in member_uids: if not self.__user_exists(member_uid): - raise exception.NotFound("Project can't be created " - "because user %s doesn't exist" % member_uid) + raise exception.NotFound("Project can't be created " + "because user %s doesn't exist" + % member_uid) members.append(self.__uid_to_dn(member_uid)) # always add the manager as a member because members is required if not manager_dn in members: From aa032cd70b3a0e312727b993d206a16a366d6ca4 Mon Sep 17 00:00:00 2001 From: Devin Carlen Date: Tue, 10 Aug 2010 19:01:40 -0700 Subject: [PATCH 37/64] Fixed admin api for user roles --- nova/adminclient.py | 13 +++++++++---- nova/endpoint/admin.py | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index 5aa8ff9c..242298a7 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -217,11 +217,16 @@ class NovaAdminClient(object): """ deletes a user """ return self.apiconn.get_object('DeregisterUser', {'Name': username}, UserInfo) + def get_roles(self, project_roles=True): + """Returns a list of available roles.""" + return self.apiconn.get_list('DescribeRoles', + {'ProjectRoles': project_roles}, + [('item', UserRole)]) + def get_user_roles(self, user, project=None): - """ - Returns a list of roles for the given user. - Omitting project will return any global roles that the user has. - Specifying project will return only project specific roles. + """Returns a list of roles for the given user. + Omitting project will return any global roles that the user has. + Specifying project will return only project specific roles. """ params = {'User':user} if project: diff --git a/nova/endpoint/admin.py b/nova/endpoint/admin.py index a3114c0a..4f4824fc 100644 --- a/nova/endpoint/admin.py +++ b/nova/endpoint/admin.py @@ -105,7 +105,8 @@ class AdminController(object): @admin_only def describe_roles(self, context, project_roles=True, **kwargs): """Returns a list of allowed roles.""" - return manager.AuthManager().get_roles(project_roles) + roles = manager.AuthManager().get_roles(project_roles) + return { 'roles': [{'role': r} for r in roles]} @admin_only def describe_user_roles(self, context, user, project=None, **kwargs): @@ -113,7 +114,8 @@ class AdminController(object): Omitting project will return any global roles that the user has. Specifying project will return only project specific roles. """ - return manager.AuthManager().get_user_roles(user, project=project) + roles = manager.AuthManager().get_user_roles(user, project=project) + return { 'roles': [{'role': r} for r in roles]} @admin_only def modify_user_role(self, context, user, role, project=None, From 0360f207e04233e55616201d7344ed50e6109f9d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 10 Aug 2010 21:51:18 -0700 Subject: [PATCH 38/64] support a hostname that can be looked up --- bin/nova-dhcpbridge | 6 ++---- nova/endpoint/cloud.py | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 6a9115fc..0dac2672 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -69,10 +69,8 @@ def init_leases(interface): """Get the list of hosts for an interface.""" net = model.get_network_by_interface(interface) res = "" - for host_name in net.hosts: - res += "%s\n" % linux_net.host_dhcp(net, - host_name, - net.hosts[host_name]) + for fixed_ip in net.hosts: + res += "%s\n" % linux_net.host_dhcp(fixed_ip, net.hosts[fixed_ip]) return res diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 02969c8e..26071cfe 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -137,17 +137,17 @@ class CloudController(object): 'root': '/dev/sda1', 'swap': 'sda3' }, - 'hostname': i['private_dns_name'], # is this public sometimes? + 'hostname': i.hostname, 'instance-action': 'none', 'instance-id': i['instance_id'], 'instance-type': i.get('instance_type', ''), - 'local-hostname': i['private_dns_name'], + 'local-hostname': i.hostname, 'local-ipv4': i['private_dns_name'], # TODO: switch to IP 'kernel-id': i.get('kernel_id', ''), 'placement': { 'availaibility-zone': i.get('availability_zone', 'nova'), }, - 'public-hostname': i.get('dns_name', ''), + 'public-hostname': i.hostname, 'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP 'public-keys': keys, 'ramdisk-id': i.get('ramdisk_id', ''), From 7c2f4df7e0650501589fee7bac3cd1ec8687b5e3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 11 Aug 2010 01:20:21 -0700 Subject: [PATCH 39/64] Actually pass in hostname and create a proper model for data in network code --- bin/nova-dhcpbridge | 4 ++-- nova/datastore.py | 12 ++++++++---- nova/endpoint/cloud.py | 18 +++++++++++++----- nova/tests/network_unittest.py | 7 ++++--- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 0dac2672..b1ad1c8f 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -69,8 +69,8 @@ def init_leases(interface): """Get the list of hosts for an interface.""" net = model.get_network_by_interface(interface) res = "" - for fixed_ip in net.hosts: - res += "%s\n" % linux_net.host_dhcp(fixed_ip, net.hosts[fixed_ip]) + for address in net.address_objs: + res += "%s\n" % linux_net.host_dhcp(address) return res diff --git a/nova/datastore.py b/nova/datastore.py index 51ef7a75..926e41f6 100644 --- a/nova/datastore.py +++ b/nova/datastore.py @@ -124,12 +124,16 @@ class BasicModel(object): yield cls(identifier) @classmethod - @absorb_connection_error def associated_to(cls, foreign_type, foreign_id): - redis_set = cls._redis_association_name(foreign_type, foreign_id) - for identifier in Redis.instance().smembers(redis_set): + for identifier in cls.associated_keys(foreign_type, foreign_id): yield cls(identifier) + @classmethod + @absorb_connection_error + def associated_keys(cls, foreign_type, foreign_id): + redis_set = cls._redis_association_name(foreign_type, foreign_id) + return Redis.instance().smembers(redis_set) or [] + @classmethod def _redis_set_name(cls, kls_name): # stupidly pluralize (for compatiblity with previous codebase) @@ -138,7 +142,7 @@ class BasicModel(object): @classmethod def _redis_association_name(cls, foreign_type, foreign_id): return cls._redis_set_name("%s:%s:%s" % - (foreign_type, foreign_id, cls.__name__)) + (foreign_type, foreign_id, cls._redis_name())) @property def identifier(self): diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 26071cfe..c79e96f5 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -125,6 +125,12 @@ class CloudController(object): } else: keys = '' + + address_record = network_model.Address(i['private_dns_name']) + if address_record: + hostname = address_record['hostname'] + else: + hostname = 'ip-%s' % i['private_dns_name'].replace('.', '-') data = { 'user-data': base64.b64decode(i['user_data']), 'meta-data': { @@ -137,17 +143,17 @@ class CloudController(object): 'root': '/dev/sda1', 'swap': 'sda3' }, - 'hostname': i.hostname, + 'hostname': hostname, 'instance-action': 'none', 'instance-id': i['instance_id'], 'instance-type': i.get('instance_type', ''), - 'local-hostname': i.hostname, + 'local-hostname': hostname, 'local-ipv4': i['private_dns_name'], # TODO: switch to IP 'kernel-id': i.get('kernel_id', ''), 'placement': { 'availaibility-zone': i.get('availability_zone', 'nova'), }, - 'public-hostname': i.hostname, + 'public-hostname': hostname, 'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP 'public-keys': keys, 'ramdisk-id': i.get('ramdisk_id', ''), @@ -563,14 +569,15 @@ class CloudController(object): is_vpn = False if image_id == FLAGS.vpn_image_id: is_vpn = True + inst = self.instdir.new() allocate_result = yield rpc.call(network_topic, {"method": "allocate_fixed_ip", "args": {"user_id": context.user.id, "project_id": context.project.id, "security_group": security_group, - "is_vpn": is_vpn}}) + "is_vpn": is_vpn, + "hostname": inst.instance_id}}) allocate_data = allocate_result['result'] - inst = self.instdir.new() inst['image_id'] = image_id inst['kernel_id'] = kernel_id inst['ramdisk_id'] = ramdisk_id @@ -584,6 +591,7 @@ class CloudController(object): inst['project_id'] = context.project.id inst['ami_launch_index'] = num inst['security_group'] = security_group + inst['hostname'] = inst.instance_id for (key, value) in allocate_data.iteritems(): inst[key] = value diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 5671a888..03950980 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -202,8 +202,8 @@ class NetworkTestCase(test.TrialTestCase): secondmac = result['mac_address'] secondaddress = result['private_dns_name'] self.assertEqual(address, secondaddress) - self.service.deallocate_fixed_ip(secondaddress) issue_ip(secondmac, secondaddress, hostname, net.bridge_name) + self.service.deallocate_fixed_ip(secondaddress) release_ip(secondmac, secondaddress, hostname, net.bridge_name) def test_available_ips(self): @@ -218,7 +218,7 @@ class NetworkTestCase(test.TrialTestCase): services (network, gateway, CloudPipe, broadcast) """ net = model.get_project_network(self.projects[0].id, "default") - num_preallocated_ips = len(net.hosts.keys()) + num_preallocated_ips = len(net.assigned) net_size = flags.FLAGS.network_size num_available_ips = net_size - (net.num_bottom_reserved_ips + num_preallocated_ips + @@ -254,7 +254,7 @@ class NetworkTestCase(test.TrialTestCase): def is_in_project(address, project_id): """Returns true if address is in specified project""" - return address in model.get_project_network(project_id).list_addresses() + return address in model.get_project_network(project_id).assigned def binpath(script): @@ -272,6 +272,7 @@ def issue_ip(mac, private_ip, hostname, interface): (out, err) = utils.execute(cmd, addl_env=env) logging.debug("ISSUE_IP: %s, %s ", out, err) + def release_ip(mac, private_ip, hostname, interface): """Run del command on dhcpbridge""" cmd = "%s del %s %s %s" % (binpath('nova-dhcpbridge'), From 95bd8036aaebbbca809285449d0230d56ab47d77 Mon Sep 17 00:00:00 2001 From: Matthew Dietz Date: Wed, 11 Aug 2010 15:27:27 -0500 Subject: [PATCH 40/64] Prototype implementation of Servers controller --- nova/endpoint/{ => aws}/cloud.py | 0 nova/endpoint/{ => aws}/images.py | 0 nova/endpoint/rackspace/controllers/base.py | 9 +++ .../endpoint/rackspace/controllers/flavors.py | 0 nova/endpoint/rackspace/controllers/images.py | 0 .../endpoint/rackspace/controllers/servers.py | 72 +++++++++++++++++++ .../rackspace/controllers/shared_ip_groups.py | 0 nova/endpoint/{ => rackspace}/rackspace.py | 27 ++++--- 8 files changed, 93 insertions(+), 15 deletions(-) rename nova/endpoint/{ => aws}/cloud.py (100%) rename nova/endpoint/{ => aws}/images.py (100%) create mode 100644 nova/endpoint/rackspace/controllers/base.py create mode 100644 nova/endpoint/rackspace/controllers/flavors.py create mode 100644 nova/endpoint/rackspace/controllers/images.py create mode 100644 nova/endpoint/rackspace/controllers/servers.py create mode 100644 nova/endpoint/rackspace/controllers/shared_ip_groups.py rename nova/endpoint/{ => rackspace}/rackspace.py (89%) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/aws/cloud.py similarity index 100% rename from nova/endpoint/cloud.py rename to nova/endpoint/aws/cloud.py diff --git a/nova/endpoint/images.py b/nova/endpoint/aws/images.py similarity index 100% rename from nova/endpoint/images.py rename to nova/endpoint/aws/images.py diff --git a/nova/endpoint/rackspace/controllers/base.py b/nova/endpoint/rackspace/controllers/base.py new file mode 100644 index 00000000..a83925cc --- /dev/null +++ b/nova/endpoint/rackspace/controllers/base.py @@ -0,0 +1,9 @@ +class BaseController(object): + @classmethod + def render(cls, instance): + if isinstance(instance, list): + return [ cls.entity_name : { cls.render(instance) } + else + return + + diff --git a/nova/endpoint/rackspace/controllers/flavors.py b/nova/endpoint/rackspace/controllers/flavors.py new file mode 100644 index 00000000..e69de29b diff --git a/nova/endpoint/rackspace/controllers/images.py b/nova/endpoint/rackspace/controllers/images.py new file mode 100644 index 00000000..e69de29b diff --git a/nova/endpoint/rackspace/controllers/servers.py b/nova/endpoint/rackspace/controllers/servers.py new file mode 100644 index 00000000..af6c958b --- /dev/null +++ b/nova/endpoint/rackspace/controllers/servers.py @@ -0,0 +1,72 @@ +from nova import rpc +from nova.compute import model as compute +from nova.endpoint.rackspace import BaseController + +class ServersController(BaseController): + entity_name = 'servers' + + def __init__(self): + raise NotImplemented("You may not create an instance of this class") + + @classmethod + def index(cls): + return [instance_details(inst) for inst in compute.InstanceDirectory().all] + + @classmethod + def show(cls, **kwargs): + instance_id = kwargs['id'] + return compute.InstanceDirectory().get(instance_id) + + @classmethod + def delete(cls, **kwargs): + instance_id = kwargs['id'] + instance = compute.InstanceDirectory().get(instance_id) + if not instance + raise ServerNotFound("The requested server was not found") + instance.destroy() + return True + + @classmethod + def create(cls, **kwargs): + inst = self.build_server_instance(kwargs['server']) + rpc.cast( + FLAGS.compute_topic, { + "method": "run_instance", + "args": {"instance_id": inst.instance_id}}) + + @classmethod + def update(cls, **kwargs): + instance_id = kwargs['id'] + instance = compute.InstanceDirectory().get(instance_id) + if not instance: + raise ServerNotFound("The requested server was not found") + instance.update(kwargs['server']) + instance.save() + + @classmethod + def build_server_instance(self, env): + """Build instance data structure and save it to the data store.""" + reservation = utils.generate_uid('r') + ltime = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + inst = self.instdir.new() + inst['name'] = env['server']['name'] + inst['image_id'] = env['server']['imageId'] + inst['instance_type'] = env['server']['flavorId'] + inst['user_id'] = env['user']['id'] + inst['project_id'] = env['project']['id'] + inst['reservation_id'] = reservation + inst['launch_time'] = ltime + inst['mac_address'] = utils.generate_mac() + address = self.network.allocate_ip( + inst['user_id'], + inst['project_id'], + mac=inst['mac_address']) + inst['private_dns_name'] = str(address) + inst['bridge_name'] = network.BridgedNetwork.get_network_for_project( + inst['user_id'], + inst['project_id'], + 'default')['bridge_name'] + # key_data, key_name, ami_launch_index + # TODO(todd): key data or root password + inst.save() + return inst diff --git a/nova/endpoint/rackspace/controllers/shared_ip_groups.py b/nova/endpoint/rackspace/controllers/shared_ip_groups.py new file mode 100644 index 00000000..e69de29b diff --git a/nova/endpoint/rackspace.py b/nova/endpoint/rackspace/rackspace.py similarity index 89% rename from nova/endpoint/rackspace.py rename to nova/endpoint/rackspace/rackspace.py index b4e6cd82..75b828e9 100644 --- a/nova/endpoint/rackspace.py +++ b/nova/endpoint/rackspace/rackspace.py @@ -45,20 +45,18 @@ class API(wsgi.Middleware): def __init__(self): super(API, self).__init__(Router(webob.exc.HTTPNotFound())) - @webob.dec.wsgify - def __call__(self, req): - return self.application + def __call__(self, environ, start_response): context = {} - if "HTTP_X_AUTH_TOKEN" in req.environ: + if "HTTP_X_AUTH_TOKEN" in environ: context['user'] = manager.AuthManager().get_user_from_access_key( - req.environ['HTTP_X_AUTH_TOKEN']) + environ['HTTP_X_AUTH_TOKEN']) if context['user']: context['project'] = manager.AuthManager().get_project( context['user'].name) if "user" not in context: - return webob.exc.HTTPForbidden() + return webob.exc.HTTPForbidden()(environ, start_response) environ['nova.context'] = context - return self.application + return self.application(environ, start_response) class Router(wsgi.Router): @@ -66,14 +64,13 @@ class Router(wsgi.Router): def _build_map(self): """Build routing map for authentication and cloud.""" - self.map.resource("server", "servers", controller=CloudServerAPI()) - #self._connect("/v1.0", controller=AuthenticationAPI()) - #cloud = CloudServerAPI() - #self._connect("/servers", controller=cloud.launch_server, - # conditions={"method": ["POST"]}) - #self._connect("/servers/{server_id}", controller=cloud.delete_server, - # conditions={'method': ["DELETE"]}) - #self._connect("/servers", controller=cloud) + self._connect("/v1.0", controller=AuthenticationAPI()) + cloud = CloudServerAPI() + self._connect("/servers", controller=cloud.launch_server, + conditions={"method": ["POST"]}) + self._connect("/servers/{server_id}", controller=cloud.delete_server, + conditions={'method': ["DELETE"]}) + self._connect("/servers", controller=cloud) class AuthenticationAPI(wsgi.Application): From 47b387ea0e8c1a57d28892cb6f76044a589158ce Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 11 Aug 2010 15:29:14 -0700 Subject: [PATCH 41/64] fix dhcpbridge issues --- bin/nova-dhcpbridge | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index b1ad1c8f..f70a4482 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -69,7 +69,7 @@ def init_leases(interface): """Get the list of hosts for an interface.""" net = model.get_network_by_interface(interface) res = "" - for address in net.address_objs: + for address in net.assigned_objs: res += "%s\n" % linux_net.host_dhcp(address) return res From 71b57c2d7764c43e55ee30d33aa3b71a20fb6e5e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 12 Aug 2010 11:52:32 -0700 Subject: [PATCH 42/64] remove syslog-ng workaround --- nova/twistd.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/nova/twistd.py b/nova/twistd.py index c83276da..8de322aa 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -241,15 +241,7 @@ def serve(filename): print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) - class NoNewlineFormatter(logging.Formatter): - """Strips newlines from default formatter""" - def format(self, record): - """Grabs default formatter's output and strips newlines""" - data = logging.Formatter.format(self, record) - return data.replace("\n", "--") - - # NOTE(vish): syslog-ng doesn't handle newlines from trackbacks very well - formatter = NoNewlineFormatter( + formatter = logging.Formatter( '(%(name)s): %(levelname)s %(message)s') handler = logging.StreamHandler(log.StdioOnnaStick()) handler.setFormatter(formatter) From ebd4e0f3bd5f2b1444fcf849fb9e420460667b61 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 12 Aug 2010 12:44:23 -0700 Subject: [PATCH 43/64] rename address stuff to avoid name collision and make the .all() iterator work again --- nova/endpoint/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index c79e96f5..27310577 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -311,7 +311,7 @@ class CloudController(object): def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py - address = network_model.PublicAddress.lookup(public_ip) + address = network_model.ElasticIp.lookup(public_ip) if address and (context.user.is_admin() or address['project_id'] == context.project.id): return address raise exception.NotFound("Address at ip %s not found" % public_ip) @@ -456,7 +456,7 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - for address in network_model.PublicAddress.all(): + for address in network_model.ElasticIp.all(): # TODO(vish): implement a by_project iterator for addresses if (context.user.is_admin() or address['project_id'] == context.project.id): From e0208d8e888f280c2f6e5b0d768cc424333c2b75 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 12 Aug 2010 13:33:22 -0700 Subject: [PATCH 44/64] typo allocated should be relased --- nova/endpoint/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index c79e96f5..1b07f2ad 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -311,7 +311,7 @@ class CloudController(object): def _get_address(self, context, public_ip): # FIXME(vish) this should move into network.py - address = network_model.PublicAddress.lookup(public_ip) + address = network_model.PublicNetworkController.get_address(public_ip) if address and (context.user.is_admin() or address['project_id'] == context.project.id): return address raise exception.NotFound("Address at ip %s not found" % public_ip) @@ -456,7 +456,7 @@ class CloudController(object): def format_addresses(self, context): addresses = [] - for address in network_model.PublicAddress.all(): + for address in network_model.PublicNetworkController.assigned_objs(): # TODO(vish): implement a by_project iterator for addresses if (context.user.is_admin() or address['project_id'] == context.project.id): From 514b11cab1ad7bb44bee9bee592e5af9d3162f30 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 12 Aug 2010 14:18:59 -0700 Subject: [PATCH 45/64] renamed missed reference to Address --- nova/endpoint/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 27310577..0a15e934 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -126,7 +126,7 @@ class CloudController(object): else: keys = '' - address_record = network_model.Address(i['private_dns_name']) + address_record = network_model.FixedIp(i['private_dns_name']) if address_record: hostname = address_record['hostname'] else: From 5cadf5670850c2106cc768d0f517a7def3b61f0d Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 12 Aug 2010 18:36:46 -0400 Subject: [PATCH 46/64] Mergeprop cleanup --- nova/endpoint/rackspace/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 nova/endpoint/rackspace/__init__.py diff --git a/nova/endpoint/rackspace/__init__.py b/nova/endpoint/rackspace/__init__.py deleted file mode 100644 index e69de29b..00000000 From e748a402460a892adec2cf2f2692011dea071f8d Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 12 Aug 2010 18:36:56 -0400 Subject: [PATCH 47/64] Mergeprop cleanup --- nova/endpoint/rackspace/{rackspace.py => __init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nova/endpoint/rackspace/{rackspace.py => __init__.py} (100%) diff --git a/nova/endpoint/rackspace/rackspace.py b/nova/endpoint/rackspace/__init__.py similarity index 100% rename from nova/endpoint/rackspace/rackspace.py rename to nova/endpoint/rackspace/__init__.py From 5eaf67a9c64dfb9a2d726e44298e7f4b8f42ee21 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 12 Aug 2010 18:41:31 -0400 Subject: [PATCH 48/64] Undo the changes to cloud.py that somehow diverged from trunk --- nova/endpoint/cloud.py | 105 ++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 878d54a1..ad9188ff 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -47,6 +47,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') + def _gen_key(user_id, key_name): """ Tuck this into AuthManager """ try: @@ -102,15 +103,16 @@ class CloudController(object): result = {} for instance in self.instdir.all: if instance['project_id'] == project_id: - line = '%s slots=%d' % (instance['private_dns_name'], INSTANCE_TYPES[instance['instance_type']]['vcpus']) + line = '%s slots=%d' % (instance['private_dns_name'], + INSTANCE_TYPES[instance['instance_type']]['vcpus']) if instance['key_name'] in result: result[instance['key_name']].append(line) else: result[instance['key_name']] = [line] return result - def get_metadata(self, ip): - i = self.get_instance_by_ip(ip) + def get_metadata(self, ipaddress): + i = self.get_instance_by_ip(ipaddress) if i is None: return None mpi = self._get_mpi_data(i['project_id']) @@ -147,7 +149,7 @@ class CloudController(object): }, 'public-hostname': i.get('dns_name', ''), 'public-ipv4': i.get('dns_name', ''), # TODO: switch to IP - 'public-keys' : keys, + 'public-keys': keys, 'ramdisk-id': i.get('ramdisk_id', ''), 'reservation-id': i['reservation_id'], 'security-groups': i.get('groups', ''), @@ -203,26 +205,22 @@ class CloudController(object): 'keyFingerprint': key_pair.fingerprint, }) - return { 'keypairsSet': result } + return {'keypairsSet': result} @rbac.allow('all') def create_key_pair(self, context, key_name, **kwargs): - try: - d = defer.Deferred() - p = context.handler.application.settings.get('pool') - def _complete(kwargs): - if 'exception' in kwargs: - d.errback(kwargs['exception']) - return - d.callback({'keyName': key_name, - 'keyFingerprint': kwargs['fingerprint'], - 'keyMaterial': kwargs['private_key']}) - p.apply_async(_gen_key, [context.user.id, key_name], - callback=_complete) - return d - - except manager.UserError as e: - raise + dcall = defer.Deferred() + pool = context.handler.application.settings.get('pool') + def _complete(kwargs): + if 'exception' in kwargs: + dcall.errback(kwargs['exception']) + return + dcall.callback({'keyName': key_name, + 'keyFingerprint': kwargs['fingerprint'], + 'keyMaterial': kwargs['private_key']}) + pool.apply_async(_gen_key, [context.user.id, key_name], + callback=_complete) + return dcall @rbac.allow('all') def delete_key_pair(self, context, key_name, **kwargs): @@ -232,7 +230,7 @@ class CloudController(object): @rbac.allow('all') def describe_security_groups(self, context, group_names, **kwargs): - groups = { 'securityGroupSet': [] } + groups = {'securityGroupSet': []} # Stubbed for now to unblock other things. return groups @@ -251,7 +249,7 @@ class CloudController(object): instance = self._get_instance(context, instance_id[0]) return rpc.call('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "get_console_output", - "args" : {"instance_id": instance_id[0]}}) + "args": {"instance_id": instance_id[0]}}) def _get_user_id(self, context): if context and context.user: @@ -285,10 +283,10 @@ class CloudController(object): if volume['attach_status'] == 'attached': v['attachmentSet'] = [{'attachTime': volume['attach_time'], 'deleteOnTermination': volume['delete_on_termination'], - 'device' : volume['mountpoint'], - 'instanceId' : volume['instance_id'], - 'status' : 'attached', - 'volume_id' : volume['volume_id']}] + 'device': volume['mountpoint'], + 'instanceId': volume['instance_id'], + 'status': 'attached', + 'volume_id': volume['volume_id']}] else: v['attachmentSet'] = [{}] return v @@ -298,7 +296,7 @@ class CloudController(object): def create_volume(self, context, size, **kwargs): # TODO(vish): refactor this to create the volume object here and tell service to create it result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume", - "args" : {"size": size, + "args": {"size": size, "user_id": context.user.id, "project_id": context.project.id}}) # NOTE(vish): rpc returned value is in the result key in the dictionary @@ -348,15 +346,15 @@ class CloudController(object): compute_node = instance['node_name'] rpc.cast('%s.%s' % (FLAGS.compute_topic, compute_node), {"method": "attach_volume", - "args" : {"volume_id": volume_id, - "instance_id" : instance_id, - "mountpoint" : device}}) - return defer.succeed({'attachTime' : volume['attach_time'], - 'device' : volume['mountpoint'], - 'instanceId' : instance_id, - 'requestId' : context.request_id, - 'status' : volume['attach_status'], - 'volumeId' : volume_id}) + "args": {"volume_id": volume_id, + "instance_id": instance_id, + "mountpoint": device}}) + return defer.succeed({'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) @rbac.allow('projectmanager', 'sysadmin') @@ -372,18 +370,18 @@ class CloudController(object): instance = self._get_instance(context, instance_id) rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "detach_volume", - "args" : {"instance_id": instance_id, + "args": {"instance_id": instance_id, "volume_id": volume_id}}) except exception.NotFound: # If the instance doesn't exist anymore, # then we need to call detach blind volume.finish_detach() - return defer.succeed({'attachTime' : volume['attach_time'], - 'device' : volume['mountpoint'], - 'instanceId' : instance_id, - 'requestId' : context.request_id, - 'status' : volume['attach_status'], - 'volumeId' : volume_id}) + return defer.succeed({'attachTime': volume['attach_time'], + 'device': volume['mountpoint'], + 'instanceId': instance_id, + 'requestId': context.request_id, + 'status': volume['attach_status'], + 'volumeId': volume_id}) def _convert_to_set(self, lst, label): if lst == None or lst == []: @@ -425,7 +423,8 @@ class CloudController(object): i['key_name'] = instance.get('key_name', None) if context.user.is_admin(): i['key_name'] = '%s (%s, %s)' % (i['key_name'], - instance.get('project_id', None), instance.get('node_name','')) + instance.get('project_id', None), + instance.get('node_name', '')) i['product_codes_set'] = self._convert_to_set( instance.get('product_codes', None), 'product_code') i['instance_type'] = instance.get('instance_type', None) @@ -442,7 +441,7 @@ class CloudController(object): reservations[res_id] = r reservations[res_id]['instances_set'].append(i) - instance_response = {'reservationSet' : list(reservations.values()) } + instance_response = {'reservationSet': list(reservations.values())} return instance_response @rbac.allow('all') @@ -457,7 +456,7 @@ class CloudController(object): address['project_id'] == context.project.id): address_rv = { 'public_ip': address['address'], - 'instance_id' : address.get('instance_id', 'free') + 'instance_id': address.get('instance_id', 'free') } if context.user.is_admin(): address_rv['instance_id'] = "%s (%s, %s)" % ( @@ -477,7 +476,7 @@ class CloudController(object): "args": {"user_id": context.user.id, "project_id": context.project.id}}) public_ip = alloc_result['result'] - defer.returnValue({'addressSet': [{'publicIp' : public_ip}]}) + defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) @rbac.allow('netadmin') @defer.inlineCallbacks @@ -591,7 +590,7 @@ class CloudController(object): inst.save() rpc.cast(FLAGS.compute_topic, {"method": "run_instance", - "args": {"instance_id" : inst.instance_id}}) + "args": {"instance_id": inst.instance_id}}) logging.debug("Casting to node for %s's instance with IP of %s" % (context.user.name, inst['private_dns_name'])) # TODO: Make Network figure out the network name from ip. @@ -646,7 +645,7 @@ class CloudController(object): instance = self._get_instance(context, i) rpc.cast('%s.%s' % (FLAGS.compute_topic, instance['node_name']), {"method": "reboot_instance", - "args" : {"instance_id": i}}) + "args": {"instance_id": i}}) return defer.succeed(True) @rbac.allow('projectmanager', 'sysadmin') @@ -656,7 +655,7 @@ class CloudController(object): volume_node = volume['node_name'] rpc.cast('%s.%s' % (FLAGS.volume_topic, volume_node), {"method": "delete_volume", - "args" : {"volume_id": volume_id}}) + "args": {"volume_id": volume_id}}) return defer.succeed(True) @rbac.allow('all') @@ -689,9 +688,9 @@ class CloudController(object): image = images.list(context, image_id)[0] except IndexError: raise exception.ApiError('invalid id: %s' % image_id) - result = { 'image_id': image_id, 'launchPermission': [] } + result = {'image_id': image_id, 'launchPermission': []} if image['isPublic']: - result['launchPermission'].append({ 'group': 'all' }) + result['launchPermission'].append({'group': 'all'}) return defer.succeed(result) @rbac.allow('projectmanager', 'sysadmin') From ed06bb00557c5157756b869dbda747c635ba601f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 12 Aug 2010 21:27:53 -0700 Subject: [PATCH 49/64] make rpc.call propogate exception info. Includes tests --- nova/endpoint/cloud.py | 15 ++++----- nova/rpc.py | 38 +++++++++++++++++------ nova/tests/rpc_unittest.py | 62 ++++++++++++++++++++++++++++++++++++++ run_tests.py | 1 + 4 files changed, 98 insertions(+), 18 deletions(-) create mode 100644 nova/tests/rpc_unittest.py diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index ad9188ff..c32fb1f7 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -103,7 +103,7 @@ class CloudController(object): result = {} for instance in self.instdir.all: if instance['project_id'] == project_id: - line = '%s slots=%d' % (instance['private_dns_name'], + line = '%s slots=%d' % (instance['private_dns_name'], INSTANCE_TYPES[instance['instance_type']]['vcpus']) if instance['key_name'] in result: result[instance['key_name']].append(line) @@ -300,7 +300,7 @@ class CloudController(object): "user_id": context.user.id, "project_id": context.project.id}}) # NOTE(vish): rpc returned value is in the result key in the dictionary - volume = self._get_volume(context, result['result']) + volume = self._get_volume(context, result) defer.returnValue({'volumeSet': [self.format_volume(context, volume)]}) def _get_address(self, context, public_ip): @@ -423,7 +423,7 @@ class CloudController(object): i['key_name'] = instance.get('key_name', None) if context.user.is_admin(): i['key_name'] = '%s (%s, %s)' % (i['key_name'], - instance.get('project_id', None), + instance.get('project_id', None), instance.get('node_name', '')) i['product_codes_set'] = self._convert_to_set( instance.get('product_codes', None), 'product_code') @@ -471,11 +471,10 @@ class CloudController(object): @defer.inlineCallbacks def allocate_address(self, context, **kwargs): network_topic = yield self._get_network_topic(context) - alloc_result = yield rpc.call(network_topic, + public_ip = yield rpc.call(network_topic, {"method": "allocate_elastic_ip", "args": {"user_id": context.user.id, "project_id": context.project.id}}) - public_ip = alloc_result['result'] defer.returnValue({'addressSet': [{'publicIp': public_ip}]}) @rbac.allow('netadmin') @@ -516,11 +515,10 @@ class CloudController(object): """Retrieves the network host for a project""" host = network_service.get_host_for_project(context.project.id) if not host: - result = yield rpc.call(FLAGS.network_topic, + host = yield rpc.call(FLAGS.network_topic, {"method": "set_network_host", "args": {"user_id": context.user.id, "project_id": context.project.id}}) - host = result['result'] defer.returnValue('%s.%s' %(FLAGS.network_topic, host)) @rbac.allow('projectmanager', 'sysadmin') @@ -563,13 +561,12 @@ class CloudController(object): vpn = False if image_id == FLAGS.vpn_image_id: vpn = True - allocate_result = yield rpc.call(network_topic, + allocate_data = yield rpc.call(network_topic, {"method": "allocate_fixed_ip", "args": {"user_id": context.user.id, "project_id": context.project.id, "security_group": security_group, "vpn": vpn}}) - allocate_data = allocate_result['result'] inst = self.instdir.new() inst['image_id'] = image_id inst['kernel_id'] = kernel_id diff --git a/nova/rpc.py b/nova/rpc.py index 2a550c3a..e06a3e19 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -40,7 +40,7 @@ FLAGS = flags.FLAGS _log = logging.getLogger('amqplib') -_log.setLevel(logging.WARN) +_log.setLevel(logging.DEBUG) class Connection(connection.BrokerConnection): @@ -141,8 +141,8 @@ class AdapterConsumer(TopicConsumer): node_args = dict((str(k), v) for k, v in args.iteritems()) d = defer.maybeDeferred(node_func, **node_args) if msg_id: - d.addCallback(lambda rval: msg_reply(msg_id, rval)) - d.addErrback(lambda e: msg_reply(msg_id, str(e))) + d.addCallback(lambda rval: msg_reply(msg_id, rval, None)) + d.addErrback(lambda e: msg_reply(msg_id, None, e)) return @@ -174,20 +174,37 @@ class DirectPublisher(Publisher): super(DirectPublisher, self).__init__(connection=connection) -def msg_reply(msg_id, reply): +def msg_reply(msg_id, reply=None, failure=None): + if failure: + message = failure.getErrorMessage() + traceback = failure.getTraceback() + logging.error("Returning exception %s to caller", message) + logging.error(traceback) + failure = (failure.type.__name__, str(failure.value), traceback) conn = Connection.instance() publisher = DirectPublisher(connection=conn, msg_id=msg_id) - try: - publisher.send({'result': reply}) - except TypeError: + publisher.send({'result': reply, 'failure': failure}) + except Exception, exc: publisher.send( {'result': dict((k, repr(v)) - for k, v in reply.__dict__.iteritems()) + for k, v in reply.__dict__.iteritems()), + 'failure': failure }) publisher.close() +class RemoteError(exception.Error): + """signifies that a remote class has raised an exception""" + def __init__(self, type, value, traceback): + self.type = type + self.value = value + self.traceback = traceback + super(RemoteError, self).__init__("%s %s\n%s" % (type, + value, + traceback)) + + def call(topic, msg): _log.debug("Making asynchronous call...") msg_id = uuid.uuid4().hex @@ -199,7 +216,10 @@ def call(topic, msg): consumer = DirectConsumer(connection=conn, msg_id=msg_id) def deferred_receive(data, message): message.ack() - d.callback(data) + if data['failure']: + return d.errback(RemoteError(*data['failure'])) + else: + return d.callback(data['result']) consumer.register_callback(deferred_receive) injected = consumer.attach_to_tornado() diff --git a/nova/tests/rpc_unittest.py b/nova/tests/rpc_unittest.py new file mode 100644 index 00000000..9c2e2934 --- /dev/null +++ b/nova/tests/rpc_unittest.py @@ -0,0 +1,62 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from twisted.internet import defer + +from nova import flags +from nova import rpc +from nova import test + + +FLAGS = flags.FLAGS + + +class RpcTestCase(test.BaseTestCase): + def setUp(self): + super(RpcTestCase, self).setUp() + self.conn = rpc.Connection.instance() + self.receiver = TestReceiver() + self.consumer = rpc.AdapterConsumer(connection=self.conn, + topic='test', + proxy=self.receiver) + + self.injected.append(self.consumer.attach_to_tornado(self.ioloop)) + + def test_call_succeed(self): + value = 42 + result = yield rpc.call('test', {"method": "echo", "args": {"value": value}}) + self.assertEqual(value, result) + + def test_call_exception(self): + value = 42 + self.assertFailure(rpc.call('test', {"method": "fail", "args": {"value": value}}), rpc.RemoteError) + try: + yield rpc.call('test', {"method": "fail", "args": {"value": value}}) + self.fail("should have thrown rpc.RemoteError") + except rpc.RemoteError as exc: + self.assertEqual(int(exc.value), value) + +class TestReceiver(object): + def echo(self, value): + logging.debug("Received %s", value) + return defer.succeed(value) + + def fail(self, value): + raise Exception(value) diff --git a/run_tests.py b/run_tests.py index 7fe6e73e..d90ac817 100644 --- a/run_tests.py +++ b/run_tests.py @@ -59,6 +59,7 @@ from nova.tests.model_unittest import * from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * from nova.tests.process_unittest import * +from nova.tests.rpc_unittest import * from nova.tests.validator_unittest import * from nova.tests.volume_unittest import * From 2d8c53b39fbdc570787bf5273f57b411458555c0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 13 Aug 2010 12:51:38 -0700 Subject: [PATCH 50/64] pep8 and pylint cleanup --- nova/rpc.py | 123 +++++++++++++++++++++++++++---------- nova/tests/rpc_unittest.py | 37 ++++++++--- 2 files changed, 122 insertions(+), 38 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index e06a3e19..4ac546c2 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -21,14 +21,13 @@ AMQP-based RPC. Queues have consumers and publishers. No fan-out support yet. """ -from carrot import connection +from carrot import connection as carrot_connection from carrot import messaging import json import logging import sys import uuid from twisted.internet import defer -from twisted.internet import reactor from twisted.internet import task from nova import exception @@ -39,13 +38,15 @@ from nova import flags FLAGS = flags.FLAGS -_log = logging.getLogger('amqplib') -_log.setLevel(logging.DEBUG) +LOG = logging.getLogger('amqplib') +LOG.setLevel(logging.DEBUG) -class Connection(connection.BrokerConnection): +class Connection(carrot_connection.BrokerConnection): + """Connection instance object""" @classmethod def instance(cls): + """Returns the instance""" if not hasattr(cls, '_instance'): params = dict(hostname=FLAGS.rabbit_host, port=FLAGS.rabbit_port, @@ -56,18 +57,33 @@ class Connection(connection.BrokerConnection): if FLAGS.fake_rabbit: params['backend_cls'] = fakerabbit.Backend + # NOTE(vish): magic is fun! + # pylint: disable=W0142 cls._instance = cls(**params) return cls._instance @classmethod def recreate(cls): + """Recreates the connection instance + + This is necessary to recover from some network errors/disconnects""" del cls._instance return cls.instance() + class Consumer(messaging.Consumer): + """Consumer base class + + Contains methods for connecting the fetch method to async loops + """ + def __init__(self, *args, **kwargs): + self.failed_connection = False + super(Consumer, self).__init__(*args, **kwargs) + # TODO(termie): it would be nice to give these some way of automatically # cleaning up after themselves def attach_to_tornado(self, io_inst=None): + """Attach a callback to tornado that fires 10 times a second""" from tornado import ioloop if io_inst is None: io_inst = ioloop.IOLoop.instance() @@ -79,33 +95,44 @@ class Consumer(messaging.Consumer): attachToTornado = attach_to_tornado - def fetch(self, *args, **kwargs): + def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): + """Wraps the parent fetch with some logic for failed connections""" # TODO(vish): the logic for failed connections and logging should be # refactored into some sort of connection manager object try: - if getattr(self, 'failed_connection', False): - # attempt to reconnect + if self.failed_connection: + # NOTE(vish): conn is defined in the parent class, we can + # recreate it as long as we create the backend too + # pylint: disable=W0201 self.conn = Connection.recreate() self.backend = self.conn.create_backend() - super(Consumer, self).fetch(*args, **kwargs) - if getattr(self, 'failed_connection', False): + super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) + if self.failed_connection: logging.error("Reconnected to queue") self.failed_connection = False - except Exception, ex: - if not getattr(self, 'failed_connection', False): + # NOTE(vish): This is catching all errors because we really don't + # exceptions to be logged 10 times a second if some + # persistent failure occurs. + except Exception: # pylint: disable=W0703 + if not self.failed_connection: logging.exception("Failed to fetch message from queue") self.failed_connection = True def attach_to_twisted(self): + """Attach a callback to twisted that fires 10 times a second""" loop = task.LoopingCall(self.fetch, enable_callbacks=True) loop.start(interval=0.1) + class Publisher(messaging.Publisher): + """Publisher base class""" pass class TopicConsumer(Consumer): + """Consumes messages on a specific topic""" exchange_type = "topic" + def __init__(self, connection=None, topic="broadcast"): self.queue = topic self.routing_key = topic @@ -115,14 +142,24 @@ class TopicConsumer(Consumer): class AdapterConsumer(TopicConsumer): + """Calls methods on a proxy object based on method and args""" def __init__(self, connection=None, topic="broadcast", proxy=None): - _log.debug('Initing the Adapter Consumer for %s' % (topic)) + LOG.debug('Initing the Adapter Consumer for %s' % (topic)) self.proxy = proxy - super(AdapterConsumer, self).__init__(connection=connection, topic=topic) + super(AdapterConsumer, self).__init__(connection=connection, + topic=topic) @exception.wrap_exception def receive(self, message_data, message): - _log.debug('received %s' % (message_data)) + """Magically looks for a method on the proxy object and calls it + + Message data should be a dictionary with two keys: + method: string representing the method to call + args: dictionary of arg: value + + Example: {'method': 'echo', 'args': {'value': 42}} + """ + LOG.debug('received %s' % (message_data)) msg_id = message_data.pop('_msg_id', None) method = message_data.get('method') @@ -133,12 +170,14 @@ class AdapterConsumer(TopicConsumer): # messages stay in the queue indefinitely, so for now # we just log the message and send an error string # back to the caller - _log.warn('no method for message: %s' % (message_data)) + LOG.warn('no method for message: %s' % (message_data)) msg_reply(msg_id, 'No method for message: %s' % message_data) return node_func = getattr(self.proxy, str(method)) node_args = dict((str(k), v) for k, v in args.iteritems()) + # NOTE(vish): magic is fun! + # pylint: disable=W0142 d = defer.maybeDeferred(node_func, **node_args) if msg_id: d.addCallback(lambda rval: msg_reply(msg_id, rval, None)) @@ -147,7 +186,9 @@ class AdapterConsumer(TopicConsumer): class TopicPublisher(Publisher): + """Publishes messages on a specific topic""" exchange_type = "topic" + def __init__(self, connection=None, topic="broadcast"): self.routing_key = topic self.exchange = FLAGS.control_exchange @@ -156,7 +197,9 @@ class TopicPublisher(Publisher): class DirectConsumer(Consumer): + """Consumes messages directly on a channel specified by msg_id""" exchange_type = "direct" + def __init__(self, connection=None, msg_id=None): self.queue = msg_id self.routing_key = msg_id @@ -166,7 +209,9 @@ class DirectConsumer(Consumer): class DirectPublisher(Publisher): + """Publishes messages directly on a channel specified by msg_id""" exchange_type = "direct" + def __init__(self, connection=None, msg_id=None): self.routing_key = msg_id self.exchange = msg_id @@ -175,51 +220,62 @@ class DirectPublisher(Publisher): def msg_reply(msg_id, reply=None, failure=None): + """Sends a reply or an error on the channel signified by msg_id + + failure should be a twisted failure object""" if failure: message = failure.getErrorMessage() traceback = failure.getTraceback() logging.error("Returning exception %s to caller", message) logging.error(traceback) - failure = (failure.type.__name__, str(failure.value), traceback) + failure = (failure.type.__name__, str(failure.value), traceback) conn = Connection.instance() publisher = DirectPublisher(connection=conn, msg_id=msg_id) try: publisher.send({'result': reply, 'failure': failure}) - except Exception, exc: + except TypeError: publisher.send( {'result': dict((k, repr(v)) for k, v in reply.__dict__.iteritems()), - 'failure': failure - }) + 'failure': failure}) publisher.close() class RemoteError(exception.Error): - """signifies that a remote class has raised an exception""" - def __init__(self, type, value, traceback): - self.type = type + """Signifies that a remote class has raised an exception + + Containes a string representation of the type of the original exception, + the value of the original exception, and the traceback. These are + sent to the parent as a joined string so printing the exception + contains all of the relevent info.""" + def __init__(self, exc_type, value, traceback): + self.exc_type = exc_type self.value = value self.traceback = traceback - super(RemoteError, self).__init__("%s %s\n%s" % (type, + super(RemoteError, self).__init__("%s %s\n%s" % (exc_type, value, traceback)) def call(topic, msg): - _log.debug("Making asynchronous call...") + """Sends a message on a topic and wait for a response""" + LOG.debug("Making asynchronous call...") msg_id = uuid.uuid4().hex msg.update({'_msg_id': msg_id}) - _log.debug("MSG_ID is %s" % (msg_id)) + LOG.debug("MSG_ID is %s" % (msg_id)) conn = Connection.instance() d = defer.Deferred() consumer = DirectConsumer(connection=conn, msg_id=msg_id) + def deferred_receive(data, message): + """Acks message and callbacks or errbacks""" message.ack() if data['failure']: return d.errback(RemoteError(*data['failure'])) else: return d.callback(data['result']) + consumer.register_callback(deferred_receive) injected = consumer.attach_to_tornado() @@ -233,7 +289,8 @@ def call(topic, msg): def cast(topic, msg): - _log.debug("Making asynchronous cast...") + """Sends a message on a topic without waiting for a response""" + LOG.debug("Making asynchronous cast...") conn = Connection.instance() publisher = TopicPublisher(connection=conn, topic=topic) publisher.send(msg) @@ -241,16 +298,18 @@ def cast(topic, msg): def generic_response(message_data, message): - _log.debug('response %s', message_data) + """Logs a result and exits""" + LOG.debug('response %s', message_data) message.ack() sys.exit(0) def send_message(topic, message, wait=True): + """Sends a message for testing""" msg_id = uuid.uuid4().hex message.update({'_msg_id': msg_id}) - _log.debug('topic is %s', topic) - _log.debug('message %s', message) + LOG.debug('topic is %s', topic) + LOG.debug('message %s', message) if wait: consumer = messaging.Consumer(connection=Connection.instance(), @@ -273,6 +332,8 @@ def send_message(topic, message, wait=True): consumer.wait() -# TODO: Replace with a docstring test if __name__ == "__main__": + # NOTE(vish): you can send messages from the command line using + # topic and a json sting representing a dictionary + # for the method send_message(sys.argv[1], json.loads(sys.argv[2])) diff --git a/nova/tests/rpc_unittest.py b/nova/tests/rpc_unittest.py index 9c2e2934..764a9741 100644 --- a/nova/tests/rpc_unittest.py +++ b/nova/tests/rpc_unittest.py @@ -15,7 +15,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +""" +Unit Tests for remote procedure calls using queue +""" import logging from twisted.internet import defer @@ -29,7 +31,8 @@ FLAGS = flags.FLAGS class RpcTestCase(test.BaseTestCase): - def setUp(self): + """Test cases for rpc""" + def setUp(self): # pylint: disable=C0103 super(RpcTestCase, self).setUp() self.conn = rpc.Connection.instance() self.receiver = TestReceiver() @@ -40,23 +43,43 @@ class RpcTestCase(test.BaseTestCase): self.injected.append(self.consumer.attach_to_tornado(self.ioloop)) def test_call_succeed(self): + """Get a value through rpc call""" value = 42 - result = yield rpc.call('test', {"method": "echo", "args": {"value": value}}) + result = yield rpc.call('test', {"method": "echo", + "args": {"value": value}}) self.assertEqual(value, result) def test_call_exception(self): + """Test that exception gets passed back properly + + rpc.call returns a RemoteError object. The value of the + exception is converted to a string, so we convert it back + to an int in the test. + """ value = 42 - self.assertFailure(rpc.call('test', {"method": "fail", "args": {"value": value}}), rpc.RemoteError) + self.assertFailure(rpc.call('test', {"method": "fail", + "args": {"value": value}}), + rpc.RemoteError) try: - yield rpc.call('test', {"method": "fail", "args": {"value": value}}) + yield rpc.call('test', {"method": "fail", + "args": {"value": value}}) self.fail("should have thrown rpc.RemoteError") except rpc.RemoteError as exc: self.assertEqual(int(exc.value), value) + class TestReceiver(object): - def echo(self, value): + """Simple Proxy class so the consumer has methods to call + + Uses static methods because we aren't actually storing any state""" + + @staticmethod + def echo(value): + """Simply returns whatever value is sent in""" logging.debug("Received %s", value) return defer.succeed(value) - def fail(self, value): + @staticmethod + def fail(value): + """Raises an exception with the value sent in""" raise Exception(value) From eb9e69fe955ab2684622efbdf0c62db81e573270 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 13 Aug 2010 14:09:30 -0700 Subject: [PATCH 51/64] Fixes out of order arguments in get_credentials --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 6af09292..071436b1 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -206,7 +206,7 @@ class ProjectCommands(object): def zipfile(self, project_id, user_id, filename='nova.zip'): """Exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" - zip_file = self.manager.get_credentials(project_id, user_id) + zip_file = self.manager.get_credentials(user_id, project_id) with open(filename, 'w') as f: f.write(zip_file) From c153bc9422308599d1354abf782273ca7bd78952 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 13 Aug 2010 23:46:44 +0200 Subject: [PATCH 52/64] Add a few unit tests for libvirt_conn. --- nova/tests/virt_unittest.py | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 nova/tests/virt_unittest.py diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py new file mode 100644 index 00000000..2aab1680 --- /dev/null +++ b/nova/tests/virt_unittest.py @@ -0,0 +1,69 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2010 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import flags +from nova import test +from nova.virt import libvirt_conn + +FLAGS = flags.FLAGS + + +class LibvirtConnTestCase(test.TrialTestCase): + def test_get_uri_and_template(self): + class MockDataModel(object): + def __init__(self): + self.datamodel = { 'name' : 'i-cafebabe', + 'memory_kb' : '1024000', + 'basepath' : '/some/path', + 'bridge_name' : 'br100', + 'mac_address' : '02:12:34:46:56:67', + 'vcpus' : 2 } + + type_uri_map = { 'qemu' : ('qemu:///system', + [lambda s: '' in s, + lambda s: 'type>hvm/usr/bin/kvm' not in s]), + 'kvm' : ('qemu:///system', + [lambda s: '' in s, + lambda s: 'type>hvm/usr/bin/qemu<' not in s]), + 'uml' : ('uml:///system', + [lambda s: '' in s, + lambda s: 'type>uml Date: Sat, 14 Aug 2010 19:04:19 +0100 Subject: [PATCH 53/64] Bug 617913: RunInstances response doesn't meet EC2 specification Fix the RunInstances response to match the EC2 specification. This involved moving the instance details from to . --- nova/endpoint/cloud.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index ad9188ff..eb0c0522 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -392,7 +392,15 @@ class CloudController(object): @rbac.allow('all') def describe_instances(self, context, **kwargs): - return defer.succeed(self._format_instances(context)) + return defer.succeed(self._format_describe_instances(context)) + + def _format_describe_instances(self, context): + return { 'reservationSet': self._format_instances(context) } + + def _format_run_instances(self, context, reservation_id): + i = self._format_instances(context, reservation_id) + assert len(i) == 1 + return i[0] def _format_instances(self, context, reservation_id = None): reservations = {} @@ -441,8 +449,7 @@ class CloudController(object): reservations[res_id] = r reservations[res_id]['instances_set'].append(i) - instance_response = {'reservationSet': list(reservations.values())} - return instance_response + return list(reservations.values()) @rbac.allow('all') def describe_addresses(self, context, **kwargs): @@ -594,7 +601,7 @@ class CloudController(object): logging.debug("Casting to node for %s's instance with IP of %s" % (context.user.name, inst['private_dns_name'])) # TODO: Make Network figure out the network name from ip. - defer.returnValue(self._format_instances(context, reservation_id)) + defer.returnValue(self._format_run_instances(context, reservation_id)) @rbac.allow('projectmanager', 'sysadmin') @defer.inlineCallbacks From 71f13bfd9bc1671a6e7693cf4f1bbb8c52a1a3c4 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sat, 14 Aug 2010 23:23:03 +0100 Subject: [PATCH 55/64] Update cloud_unittest to match renamed internal function. --- nova/tests/cloud_unittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 40837405..3501771c 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -132,7 +132,7 @@ class CloudTestCase(test.BaseTestCase): 'state': 0x01, 'user_data': '' } - rv = self.cloud._format_instances(self.context) + rv = self.cloud._format_describe_instances(self.context) self.assert_(len(rv['reservationSet']) == 0) # simulate launch of 5 instances From 02b35b0c17a7e5cfec419fdb4ed92bdcf3503810 Mon Sep 17 00:00:00 2001 From: andy Date: Mon, 16 Aug 2010 14:16:21 +0200 Subject: [PATCH 56/64] rather comprehensive style fixes --- nova/adminclient.py | 7 +++++++ nova/auth/fakeldap.py | 1 - nova/auth/ldapdriver.py | 1 + nova/auth/manager.py | 12 +++++++----- nova/auth/rbac.py | 2 ++ nova/auth/signer.py | 10 +++++++--- nova/endpoint/admin.py | 4 ++++ nova/endpoint/api.py | 7 +++++-- nova/endpoint/cloud.py | 3 +-- nova/endpoint/images.py | 7 ++++++- nova/fakerabbit.py | 5 +++-- nova/flags.py | 38 ++++++++++++++++---------------------- nova/process.py | 3 +++ nova/rpc.py | 5 +++-- nova/validate.py | 1 + run_tests.py | 9 ++++----- 16 files changed, 70 insertions(+), 45 deletions(-) diff --git a/nova/adminclient.py b/nova/adminclient.py index 242298a7..0ca32b1e 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -20,6 +20,7 @@ Nova User API client library. """ import base64 + import boto from boto.ec2.regioninfo import RegionInfo @@ -57,6 +58,7 @@ class UserInfo(object): elif name == 'secretkey': self.secretkey = str(value) + class UserRole(object): """ Information about a Nova user's role, as parsed through SAX. @@ -79,6 +81,7 @@ class UserRole(object): else: setattr(self, name, str(value)) + class ProjectInfo(object): """ Information about a Nova project, as parsed through SAX @@ -114,12 +117,14 @@ class ProjectInfo(object): else: setattr(self, name, str(value)) + class ProjectMember(object): """ Information about a Nova project member, as parsed through SAX. Fields include: memberId """ + def __init__(self, connection=None): self.connection = connection self.memberId = None @@ -135,6 +140,7 @@ class ProjectMember(object): self.memberId = value else: setattr(self, name, str(value)) + class HostInfo(object): """ @@ -163,6 +169,7 @@ class HostInfo(object): def endElement(self, name, value, connection): setattr(self, name, value) + class NovaAdminClient(object): def __init__(self, clc_ip='127.0.0.1', region='nova', access_key='admin', secret_key='admin', **kwargs): diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index b420924a..bc744fa0 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -219,7 +219,6 @@ class FakeLDAP(object): raise NO_SUCH_OBJECT() return objects - @property def __redis_prefix(self): return 'ldap:' diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 453fa196..6bf7fcd1 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -30,6 +30,7 @@ import sys from nova import exception from nova import flags + FLAGS = flags.FLAGS flags.DEFINE_string('ldap_url', 'ldap://localhost', 'Point this at your ldap server') diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 064fd78b..80ee7889 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -37,7 +37,6 @@ from nova.network import vpn FLAGS = flags.FLAGS - flags.DEFINE_list('allowed_roles', ['cloudadmin', 'itsec', 'sysadmin', 'netadmin', 'developer'], 'Allowed roles for project') @@ -52,7 +51,6 @@ flags.DEFINE_list('superuser_roles', ['cloudadmin'], flags.DEFINE_list('global_roles', ['cloudadmin', 'itsec'], 'Roles that apply to all projects') - flags.DEFINE_string('credentials_template', utils.abspath('auth/novarc.template'), 'Template for creating users rc file') @@ -67,15 +65,14 @@ flags.DEFINE_string('credential_cert_file', 'cert.pem', 'Filename of certificate in credentials zip') flags.DEFINE_string('credential_rc_file', 'novarc', 'Filename of rc in credentials zip') - flags.DEFINE_string('credential_cert_subject', '/C=US/ST=California/L=MountainView/O=AnsoLabs/' 'OU=NovaDev/CN=%s-%s', 'Subject for certificate for users') - flags.DEFINE_string('auth_driver', 'nova.auth.ldapdriver.FakeLdapDriver', 'Driver that auth manager uses') + class AuthBase(object): """Base class for objects relating to auth @@ -83,6 +80,7 @@ class AuthBase(object): an id member. They may optionally contain methods that delegate to AuthManager, but should not implement logic themselves. """ + @classmethod def safe_id(cls, obj): """Safe get object id @@ -100,6 +98,7 @@ class AuthBase(object): class User(AuthBase): """Object representing a user""" + def __init__(self, id, name, access, secret, admin): AuthBase.__init__(self) self.id = id @@ -161,6 +160,7 @@ class KeyPair(AuthBase): Even though this object is named KeyPair, only the public key and fingerprint is stored. The user's private key is not saved. """ + def __init__(self, id, name, owner_id, public_key, fingerprint): AuthBase.__init__(self) self.id = id @@ -179,6 +179,7 @@ class KeyPair(AuthBase): class Project(AuthBase): """Represents a Project returned from the datastore""" + def __init__(self, id, name, project_manager_id, description, member_ids): AuthBase.__init__(self) self.id = id @@ -227,7 +228,6 @@ class Project(AuthBase): self.member_ids) - class AuthManager(object): """Manager Singleton for dealing with Users, Projects, and Keypairs @@ -239,7 +239,9 @@ class AuthManager(object): AuthManager also manages associated data related to Auth objects that need to be more accessible, such as vpn ips and ports. """ + _instance = None + def __new__(cls, *args, **kwargs): """Returns the AuthManager singleton""" if not cls._instance: diff --git a/nova/auth/rbac.py b/nova/auth/rbac.py index 7fab9419..1446e4e2 100644 --- a/nova/auth/rbac.py +++ b/nova/auth/rbac.py @@ -32,6 +32,7 @@ def allow(*roles): return wrapped_f return wrap + def deny(*roles): def wrap(f): def wrapped_f(self, context, *args, **kwargs): @@ -44,6 +45,7 @@ def deny(*roles): return wrapped_f return wrap + def __matches_role(context, role): if role == 'all': return True diff --git a/nova/auth/signer.py b/nova/auth/signer.py index 634f22f0..8334806d 100644 --- a/nova/auth/signer.py +++ b/nova/auth/signer.py @@ -48,11 +48,15 @@ import hashlib import hmac import logging import urllib -import boto # NOTE(vish): for new boto -import boto.utils # NOTE(vish): for old boto + +# NOTE(vish): for new boto +import boto +# NOTE(vish): for old boto +import boto.utils from nova.exception import Error + class Signer(object): """ hacked up code from boto/connection.py """ @@ -77,7 +81,6 @@ class Signer(object): return self._calc_signature_2(params, verb, server_string, path) raise Error('Unknown Signature Version: %s' % self.SignatureVersion) - def _get_utf8_value(self, value): if not isinstance(value, str) and not isinstance(value, unicode): value = str(value) @@ -133,5 +136,6 @@ class Signer(object): logging.debug('base64 encoded digest: %s' % b64) return b64 + if __name__ == '__main__': print Signer('foo').generate({"SignatureMethod": 'HmacSHA256', 'SignatureVersion': '2'}, "get", "server", "/foo") diff --git a/nova/endpoint/admin.py b/nova/endpoint/admin.py index 4f4824fc..d6f62275 100644 --- a/nova/endpoint/admin.py +++ b/nova/endpoint/admin.py @@ -37,6 +37,7 @@ def user_dict(user, base64_file=None): else: return {} + def project_dict(project): """Convert the project object to a result dict""" if project: @@ -47,6 +48,7 @@ def project_dict(project): else: return {} + def host_dict(host): """Convert a host model object to a result dict""" if host: @@ -54,6 +56,7 @@ def host_dict(host): else: return {} + def admin_only(target): """Decorator for admin-only API calls""" def wrapper(*args, **kwargs): @@ -66,6 +69,7 @@ def admin_only(target): return wrapper + class AdminController(object): """ API Controller for users, hosts, nodes, and workers. diff --git a/nova/endpoint/api.py b/nova/endpoint/api.py index 78a18b9e..40be00bb 100755 --- a/nova/endpoint/api.py +++ b/nova/endpoint/api.py @@ -25,12 +25,13 @@ import logging import multiprocessing import random import re -import tornado.web -from twisted.internet import defer import urllib # TODO(termie): replace minidom with etree from xml.dom import minidom +import tornado.web +from twisted.internet import defer + from nova import crypto from nova import exception from nova import flags @@ -43,6 +44,7 @@ from nova.endpoint import cloud FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') + _log = logging.getLogger("api") _log.setLevel(logging.DEBUG) @@ -227,6 +229,7 @@ class MetadataRequestHandler(tornado.web.RequestHandler): self.print_data(data) self.finish() + class APIRequestHandler(tornado.web.RequestHandler): def get(self, controller_name): self.execute(controller_name) diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py index 5366acec..a3d6d1aa 100644 --- a/nova/endpoint/cloud.py +++ b/nova/endpoint/cloud.py @@ -26,6 +26,7 @@ import base64 import logging import os import time + from twisted.internet import defer from nova import datastore @@ -44,7 +45,6 @@ from nova.volume import service FLAGS = flags.FLAGS - flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') @@ -362,7 +362,6 @@ class CloudController(object): 'status': volume['attach_status'], 'volumeId': volume_id}) - @rbac.allow('projectmanager', 'sysadmin') def detach_volume(self, context, volume_id, **kwargs): volume = self._get_volume(context, volume_id) diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py index fe7cb5d1..2a88d66a 100644 --- a/nova/endpoint/images.py +++ b/nova/endpoint/images.py @@ -21,10 +21,11 @@ Proxy AMI-related calls from the cloud controller, to the running objectstore daemon. """ -import boto.s3.connection import json import urllib +import boto.s3.connection + from nova import flags from nova import utils from nova.auth import manager @@ -32,6 +33,7 @@ from nova.auth import manager FLAGS = flags.FLAGS + def modify(context, image_id, operation): conn(context).make_request( method='POST', @@ -53,6 +55,7 @@ def register(context, image_location): return image_id + def list(context, filter_list=[]): """ return a list of all images that a user can see @@ -68,6 +71,7 @@ def list(context, filter_list=[]): return [i for i in result if i['imageId'] in filter_list] return result + def deregister(context, image_id): """ unregister an image """ conn(context).make_request( @@ -75,6 +79,7 @@ def deregister(context, image_id): bucket='_images', query_args=qs({'image_id': image_id})) + def conn(context): access = manager.AuthManager().get_access_key(context.user, context.project) diff --git a/nova/fakerabbit.py b/nova/fakerabbit.py index 68919451..06802524 100644 --- a/nova/fakerabbit.py +++ b/nova/fakerabbit.py @@ -16,12 +16,13 @@ # License for the specific language governing permissions and limitations # under the License. -""" Based a bit on the carrot.backeds.queue backend... but a lot better """ +"""Based a bit on the carrot.backeds.queue backend... but a lot better.""" -from carrot.backends import base import logging import Queue as queue +from carrot.backends import base + class Message(base.BaseMessage): pass diff --git a/nova/flags.py b/nova/flags.py index b3bdd088..e3feb252 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -175,29 +175,25 @@ DEFINE_string('network_topic', 'network', 'the topic network nodes listen on') DEFINE_bool('verbose', False, 'show debug output') DEFINE_boolean('fake_rabbit', False, 'use a fake rabbit') -DEFINE_bool('fake_network', False, 'should we use fake network devices and addresses') +DEFINE_bool('fake_network', False, + 'should we use fake network devices and addresses') DEFINE_string('rabbit_host', 'localhost', 'rabbit host') DEFINE_integer('rabbit_port', 5672, 'rabbit port') DEFINE_string('rabbit_userid', 'guest', 'rabbit userid') DEFINE_string('rabbit_password', 'guest', 'rabbit password') DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') -DEFINE_string('ec2_url', - 'http://127.0.0.1:8773/services/Cloud', - 'Url to ec2 api server') +DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud', + 'Url to ec2 api server') -DEFINE_string('default_image', - 'ami-11111', - 'default image to use, testing only') -DEFINE_string('default_kernel', - 'aki-11111', - 'default kernel to use, testing only') -DEFINE_string('default_ramdisk', - 'ari-11111', - 'default ramdisk to use, testing only') -DEFINE_string('default_instance_type', - 'm1.small', - 'default instance type to use, testing only') +DEFINE_string('default_image', 'ami-11111', + 'default image to use, testing only') +DEFINE_string('default_kernel', 'aki-11111', + 'default kernel to use, testing only') +DEFINE_string('default_ramdisk', 'ari-11111', + 'default ramdisk to use, testing only') +DEFINE_string('default_instance_type', 'm1.small', + 'default instance type to use, testing only') DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', 'AMI for cloudpipe vpn server') DEFINE_string('vpn_key_suffix', @@ -207,10 +203,8 @@ DEFINE_string('vpn_key_suffix', DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') # UNUSED -DEFINE_string('node_availability_zone', - 'nova', - 'availability zone of this node') -DEFINE_string('node_name', - socket.gethostname(), - 'name of this node') +DEFINE_string('node_availability_zone', 'nova', + 'availability zone of this node') +DEFINE_string('node_name', socket.gethostname(), + 'name of this node') diff --git a/nova/process.py b/nova/process.py index 2dc56372..86f29e2c 100644 --- a/nova/process.py +++ b/nova/process.py @@ -23,6 +23,7 @@ Process pool, still buggy right now. import logging import multiprocessing import StringIO + from twisted.internet import defer from twisted.internet import error from twisted.internet import process @@ -205,6 +206,7 @@ class ProcessPool(object): self._pool.release() return rv + class SharedPool(object): _instance = None def __init__(self): @@ -213,5 +215,6 @@ class SharedPool(object): def __getattr__(self, key): return getattr(self._instance, key) + def simple_execute(cmd, **kwargs): return SharedPool().simple_execute(cmd, **kwargs) diff --git a/nova/rpc.py b/nova/rpc.py index 4ac546c2..824a66b5 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -21,12 +21,13 @@ AMQP-based RPC. Queues have consumers and publishers. No fan-out support yet. """ -from carrot import connection as carrot_connection -from carrot import messaging import json import logging import sys import uuid + +from carrot import connection as carrot_connection +from carrot import messaging from twisted.internet import defer from twisted.internet import task diff --git a/nova/validate.py b/nova/validate.py index a69306fa..21f4ed28 100644 --- a/nova/validate.py +++ b/nova/validate.py @@ -57,6 +57,7 @@ def rangetest(**argchecks): # validate ranges for both+defaults return onCall return onDecorator + def typetest(**argchecks): def onDecorator(func): import sys diff --git a/run_tests.py b/run_tests.py index d90ac817..77aa9088 100644 --- a/run_tests.py +++ b/run_tests.py @@ -38,11 +38,11 @@ Due to our use of multiprocessing it we frequently get some ignorable 'Interrupted system call' exceptions after test completion. """ + import __main__ import os import sys - from twisted.scripts import trial as trial_script from nova import datastore @@ -65,13 +65,12 @@ from nova.tests.volume_unittest import * FLAGS = flags.FLAGS - flags.DEFINE_bool('flush_db', True, 'Flush the database before running fake tests') - flags.DEFINE_string('tests_stderr', 'run_tests.err.log', - 'Path to where to pipe STDERR during test runs. ' - 'Default = "run_tests.err.log"') + 'Path to where to pipe STDERR during test runs.' + ' Default = "run_tests.err.log"') + if __name__ == '__main__': OptionsClass = twistd.WrapTwistedOptions(trial_script.Options) From 64923941ed4b7cd3f9c69ef88eb12662ee18a74a Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 16 Aug 2010 10:57:42 -0400 Subject: [PATCH 57/64] PEP8 and name corrections --- bin/nova-rsapi | 2 +- nova/endpoint/aws/__init__.py | 4 ++-- nova/endpoint/rackspace/__init__.py | 10 +++++----- nova/endpoint/rackspace/controllers/base.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/nova-rsapi b/bin/nova-rsapi index 3fc61860..a35936ef 100755 --- a/bin/nova-rsapi +++ b/bin/nova-rsapi @@ -32,4 +32,4 @@ flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') if __name__ == '__main__': utils.default_flagfile() - wsgi.run_server(nova.endpoint.ApiVersionRouter(), FLAGS.cc_port) + wsgi.run_server(nova.endpoint.APIVersionRouter(), FLAGS.cc_port) diff --git a/nova/endpoint/aws/__init__.py b/nova/endpoint/aws/__init__.py index f49270a3..4507cae6 100644 --- a/nova/endpoint/aws/__init__.py +++ b/nova/endpoint/aws/__init__.py @@ -4,7 +4,7 @@ import webob.dec from nova import wsgi # TODO(gundlach): temp -class Api(wsgi.Router): +class API(wsgi.Router): """WSGI entry point for all AWS API requests.""" def __init__(self): @@ -14,7 +14,7 @@ class Api(wsgi.Router): targets = {"dummy": self.dummy } - super(Api, self).__init__(mapper, targets) + super(API, self).__init__(mapper, targets) @webob.dec.wsgify def dummy(self, req): diff --git a/nova/endpoint/rackspace/__init__.py b/nova/endpoint/rackspace/__init__.py index f14f6218..162b35ca 100644 --- a/nova/endpoint/rackspace/__init__.py +++ b/nova/endpoint/rackspace/__init__.py @@ -37,12 +37,12 @@ FLAGS = flags.FLAGS flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') -class Api(wsgi.Middleware): +class API(wsgi.Middleware): """WSGI entry point for all Rackspace API requests.""" def __init__(self): - app = AuthMiddleware(ApiRouter()) - super(Api, self).__init__(app) + app = AuthMiddleware(APIRouter()) + super(API, self).__init__(app) class AuthMiddleware(wsgi.Middleware): @@ -66,7 +66,7 @@ class AuthMiddleware(wsgi.Middleware): return self.application -class ApiRouter(wsgi.Router): +class APIRouter(wsgi.Router): """ Routes requests on the Rackspace API to the appropriate controller and method. @@ -87,4 +87,4 @@ class ApiRouter(wsgi.Router): 'sharedipgroups': controllers.SharedIpGroupsController() } - super(ApiRouter, self).__init__(mapper, targets) + super(APIRouter, self).__init__(mapper, targets) diff --git a/nova/endpoint/rackspace/controllers/base.py b/nova/endpoint/rackspace/controllers/base.py index 3ada53fd..8cd44f62 100644 --- a/nova/endpoint/rackspace/controllers/base.py +++ b/nova/endpoint/rackspace/controllers/base.py @@ -1,6 +1,6 @@ -from nova.wsgi import WSGIController +from nova import wsgi -class BaseController(WSGIController): +class BaseController(wsgi.Controller): @classmethod def render(cls, instance): if isinstance(instance, list): From 83ba85655272ceb84b6df1abf7d7bd01f43303d1 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 16 Aug 2010 13:22:41 -0400 Subject: [PATCH 58/64] All controller actions receive a 'req' parameter containing the webob Request. --- nova/endpoint/__init__.py | 10 ++++----- nova/endpoint/aws/__init__.py | 6 ++---- nova/endpoint/rackspace/__init__.py | 21 ++++++++----------- .../endpoint/rackspace/controllers/servers.py | 2 +- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/nova/endpoint/__init__.py b/nova/endpoint/__init__.py index 065f4584..9aae933a 100644 --- a/nova/endpoint/__init__.py +++ b/nova/endpoint/__init__.py @@ -36,16 +36,16 @@ import routes from nova.endpoint import rackspace from nova.endpoint import aws -class ApiVersionRouter(wsgi.Router): +class APIVersionRouter(wsgi.Router): """Routes top-level requests to the appropriate API.""" def __init__(self): mapper = routes.Mapper() - mapper.connect(None, "/v1.0/{path_info:.*}", controller="rs") - mapper.connect(None, "/ec2/{path_info:.*}", controller="ec2") + rsapi = rackspace.API() + mapper.connect(None, "/v1.0/{path_info:.*}", controller=rsapi) - targets = {"rs": rackspace.Api(), "ec2": aws.Api()} + mapper.connect(None, "/ec2/{path_info:.*}", controller=aws.API()) - super(ApiVersionRouter, self).__init__(mapper, targets) + super(APIVersionRouter, self).__init__(mapper) diff --git a/nova/endpoint/aws/__init__.py b/nova/endpoint/aws/__init__.py index 4507cae6..55cbb8fd 100644 --- a/nova/endpoint/aws/__init__.py +++ b/nova/endpoint/aws/__init__.py @@ -10,11 +10,9 @@ class API(wsgi.Router): def __init__(self): mapper = routes.Mapper() - mapper.connect(None, "{all:.*}", controller="dummy") + mapper.connect(None, "{all:.*}", controller=self.dummy) - targets = {"dummy": self.dummy } - - super(API, self).__init__(mapper, targets) + super(API, self).__init__(mapper) @webob.dec.wsgify def dummy(self, req): diff --git a/nova/endpoint/rackspace/__init__.py b/nova/endpoint/rackspace/__init__.py index 162b35ca..78b9c942 100644 --- a/nova/endpoint/rackspace/__init__.py +++ b/nova/endpoint/rackspace/__init__.py @@ -75,16 +75,13 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() - mapper.resource("server", "servers") - mapper.resource("image", "images") - mapper.resource("flavor", "flavors") - mapper.resource("sharedipgroup", "sharedipgroups") + mapper.resource("server", "servers", + controller=controllers.ServersController()) + mapper.resource("image", "images", + controller=controllers.ImagesController()) + mapper.resource("flavor", "flavors", + controller=controllers.FlavorsController()) + mapper.resource("sharedipgroup", "sharedipgroups", + controller=controllers.SharedIpGroupsController()) - targets = { - 'servers': controllers.ServersController(), - 'images': controllers.ImagesController(), - 'flavors': controllers.FlavorsController(), - 'sharedipgroups': controllers.SharedIpGroupsController() - } - - super(APIRouter, self).__init__(mapper, targets) + super(APIRouter, self).__init__(mapper) diff --git a/nova/endpoint/rackspace/controllers/servers.py b/nova/endpoint/rackspace/controllers/servers.py index db02e058..2f8e662d 100644 --- a/nova/endpoint/rackspace/controllers/servers.py +++ b/nova/endpoint/rackspace/controllers/servers.py @@ -5,7 +5,7 @@ from nova.endpoint.rackspace.controllers.base import BaseController class ServersController(BaseController): entity_name = 'servers' - def index(cls): + def index(self, **kwargs): return [instance_details(inst) for inst in compute.InstanceDirectory().all] def show(self, **kwargs): From 17ce9090eb06aa36a7d19f4b816370fb0b91db7a Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 17 Aug 2010 13:03:38 -0400 Subject: [PATCH 60/64] Remove duplicate definition of flag --- nova/endpoint/rackspace/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/endpoint/rackspace/__init__.py b/nova/endpoint/rackspace/__init__.py index 78b9c942..ac53ee10 100644 --- a/nova/endpoint/rackspace/__init__.py +++ b/nova/endpoint/rackspace/__init__.py @@ -33,10 +33,6 @@ from nova.auth import manager from nova.endpoint.rackspace import controllers -FLAGS = flags.FLAGS -flags.DEFINE_string('cloud_topic', 'cloud', 'the topic clouds listen on') - - class API(wsgi.Middleware): """WSGI entry point for all Rackspace API requests.""" From 1e77a7849df50b38c215fff13e58df7e9721ffef Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 17 Aug 2010 13:23:20 -0400 Subject: [PATCH 61/64] Move eventlet-using class out of endpoint/__init__.py into its own submodule, so that twisted-related code using endpoint.[other stuff] wouldn't run eventlet and make unit tests throw crazy errors about eventlet 0.9.10 not playing nicely with twisted. --- bin/nova-rsapi | 5 ++-- nova/endpoint/__init__.py | 51 --------------------------------------- nova/endpoint/newapi.py | 51 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 54 deletions(-) create mode 100644 nova/endpoint/newapi.py diff --git a/bin/nova-rsapi b/bin/nova-rsapi index a35936ef..e2722422 100755 --- a/bin/nova-rsapi +++ b/bin/nova-rsapi @@ -21,15 +21,14 @@ Daemon for the Rackspace API endpoint. """ -import nova.endpoint - from nova import flags from nova import utils from nova import wsgi +from nova.endpoint import newapi FLAGS = flags.FLAGS flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') if __name__ == '__main__': utils.default_flagfile() - wsgi.run_server(nova.endpoint.APIVersionRouter(), FLAGS.cc_port) + wsgi.run_server(newapi.APIVersionRouter(), FLAGS.cc_port) diff --git a/nova/endpoint/__init__.py b/nova/endpoint/__init__.py index 9aae933a..e69de29b 100644 --- a/nova/endpoint/__init__.py +++ b/nova/endpoint/__init__.py @@ -1,51 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -:mod:`nova.endpoint` -- Main NOVA Api endpoints -===================================================== - -.. automodule:: nova.endpoint - :platform: Unix - :synopsis: REST APIs for all nova functions -.. moduleauthor:: Jesse Andrews -.. moduleauthor:: Devin Carlen -.. moduleauthor:: Vishvananda Ishaya -.. moduleauthor:: Joshua McKenty -.. moduleauthor:: Manish Singh -.. moduleauthor:: Andy Smith -""" - -from nova import wsgi -import routes -from nova.endpoint import rackspace -from nova.endpoint import aws - -class APIVersionRouter(wsgi.Router): - """Routes top-level requests to the appropriate API.""" - - def __init__(self): - mapper = routes.Mapper() - - rsapi = rackspace.API() - mapper.connect(None, "/v1.0/{path_info:.*}", controller=rsapi) - - mapper.connect(None, "/ec2/{path_info:.*}", controller=aws.API()) - - super(APIVersionRouter, self).__init__(mapper) - diff --git a/nova/endpoint/newapi.py b/nova/endpoint/newapi.py new file mode 100644 index 00000000..9aae933a --- /dev/null +++ b/nova/endpoint/newapi.py @@ -0,0 +1,51 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`nova.endpoint` -- Main NOVA Api endpoints +===================================================== + +.. automodule:: nova.endpoint + :platform: Unix + :synopsis: REST APIs for all nova functions +.. moduleauthor:: Jesse Andrews +.. moduleauthor:: Devin Carlen +.. moduleauthor:: Vishvananda Ishaya +.. moduleauthor:: Joshua McKenty +.. moduleauthor:: Manish Singh +.. moduleauthor:: Andy Smith +""" + +from nova import wsgi +import routes +from nova.endpoint import rackspace +from nova.endpoint import aws + +class APIVersionRouter(wsgi.Router): + """Routes top-level requests to the appropriate API.""" + + def __init__(self): + mapper = routes.Mapper() + + rsapi = rackspace.API() + mapper.connect(None, "/v1.0/{path_info:.*}", controller=rsapi) + + mapper.connect(None, "/ec2/{path_info:.*}", controller=aws.API()) + + super(APIVersionRouter, self).__init__(mapper) + From 6cc3f5650741362e4dd4eefb61b223931ef57bf8 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 17 Aug 2010 16:43:37 -0700 Subject: [PATCH 62/64] In an effort to keep new and old API code separate, I've created a nova.api to put all new API code under. This means nova.endpoint only contains the old Tornado implementation. I also cleaned up a few pep8 and other style nits in the new API code. --- bin/{nova-rsapi => nova-api-new} | 8 +- nova/endpoint/aws/__init__.py | 22 ----- nova/endpoint/newapi.py | 51 ------------ nova/endpoint/rackspace/__init__.py | 83 ------------------- .../rackspace/controllers/__init__.py | 5 -- nova/endpoint/rackspace/controllers/base.py | 9 -- .../endpoint/rackspace/controllers/flavors.py | 1 - nova/endpoint/rackspace/controllers/images.py | 1 - .../endpoint/rackspace/controllers/servers.py | 63 -------------- .../rackspace/controllers/sharedipgroups.py | 1 - 10 files changed, 4 insertions(+), 240 deletions(-) rename bin/{nova-rsapi => nova-api-new} (83%) delete mode 100644 nova/endpoint/aws/__init__.py delete mode 100644 nova/endpoint/newapi.py delete mode 100644 nova/endpoint/rackspace/__init__.py delete mode 100644 nova/endpoint/rackspace/controllers/__init__.py delete mode 100644 nova/endpoint/rackspace/controllers/base.py delete mode 100644 nova/endpoint/rackspace/controllers/flavors.py delete mode 100644 nova/endpoint/rackspace/controllers/images.py delete mode 100644 nova/endpoint/rackspace/controllers/servers.py delete mode 100644 nova/endpoint/rackspace/controllers/sharedipgroups.py diff --git a/bin/nova-rsapi b/bin/nova-api-new similarity index 83% rename from bin/nova-rsapi rename to bin/nova-api-new index e2722422..fda42339 100755 --- a/bin/nova-rsapi +++ b/bin/nova-api-new @@ -18,17 +18,17 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - Daemon for the Rackspace API endpoint. +Nova API daemon. """ +from nova import api from nova import flags from nova import utils from nova import wsgi -from nova.endpoint import newapi FLAGS = flags.FLAGS -flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') +flags.DEFINE_integer('api_port', 8773, 'API port') if __name__ == '__main__': utils.default_flagfile() - wsgi.run_server(newapi.APIVersionRouter(), FLAGS.cc_port) + wsgi.run_server(api.API(), FLAGS.api_port) diff --git a/nova/endpoint/aws/__init__.py b/nova/endpoint/aws/__init__.py deleted file mode 100644 index 55cbb8fd..00000000 --- a/nova/endpoint/aws/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -import routes -import webob.dec - -from nova import wsgi - -# TODO(gundlach): temp -class API(wsgi.Router): - """WSGI entry point for all AWS API requests.""" - - def __init__(self): - mapper = routes.Mapper() - - mapper.connect(None, "{all:.*}", controller=self.dummy) - - super(API, self).__init__(mapper) - - @webob.dec.wsgify - def dummy(self, req): - #TODO(gundlach) - msg = "dummy response -- please hook up __init__() to cloud.py instead" - return repr({ 'dummy': msg, - 'kwargs': repr(req.environ['wsgiorg.routing_args'][1]) }) diff --git a/nova/endpoint/newapi.py b/nova/endpoint/newapi.py deleted file mode 100644 index 9aae933a..00000000 --- a/nova/endpoint/newapi.py +++ /dev/null @@ -1,51 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -:mod:`nova.endpoint` -- Main NOVA Api endpoints -===================================================== - -.. automodule:: nova.endpoint - :platform: Unix - :synopsis: REST APIs for all nova functions -.. moduleauthor:: Jesse Andrews -.. moduleauthor:: Devin Carlen -.. moduleauthor:: Vishvananda Ishaya -.. moduleauthor:: Joshua McKenty -.. moduleauthor:: Manish Singh -.. moduleauthor:: Andy Smith -""" - -from nova import wsgi -import routes -from nova.endpoint import rackspace -from nova.endpoint import aws - -class APIVersionRouter(wsgi.Router): - """Routes top-level requests to the appropriate API.""" - - def __init__(self): - mapper = routes.Mapper() - - rsapi = rackspace.API() - mapper.connect(None, "/v1.0/{path_info:.*}", controller=rsapi) - - mapper.connect(None, "/ec2/{path_info:.*}", controller=aws.API()) - - super(APIVersionRouter, self).__init__(mapper) - diff --git a/nova/endpoint/rackspace/__init__.py b/nova/endpoint/rackspace/__init__.py deleted file mode 100644 index ac53ee10..00000000 --- a/nova/endpoint/rackspace/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Rackspace API Endpoint -""" - -import json -import time - -import webob.dec -import webob.exc -import routes - -from nova import flags -from nova import wsgi -from nova.auth import manager -from nova.endpoint.rackspace import controllers - - -class API(wsgi.Middleware): - """WSGI entry point for all Rackspace API requests.""" - - def __init__(self): - app = AuthMiddleware(APIRouter()) - super(API, self).__init__(app) - - -class AuthMiddleware(wsgi.Middleware): - """Authorize the rackspace API request or return an HTTP Forbidden.""" - - #TODO(gundlach): isn't this the old Nova API's auth? Should it be replaced - #with correct RS API auth? - - @webob.dec.wsgify - def __call__(self, req): - context = {} - if "HTTP_X_AUTH_TOKEN" in req.environ: - context['user'] = manager.AuthManager().get_user_from_access_key( - req.environ['HTTP_X_AUTH_TOKEN']) - if context['user']: - context['project'] = manager.AuthManager().get_project( - context['user'].name) - if "user" not in context: - return webob.exc.HTTPForbidden() - req.environ['nova.context'] = context - return self.application - - -class APIRouter(wsgi.Router): - """ - Routes requests on the Rackspace API to the appropriate controller - and method. - """ - - def __init__(self): - mapper = routes.Mapper() - - mapper.resource("server", "servers", - controller=controllers.ServersController()) - mapper.resource("image", "images", - controller=controllers.ImagesController()) - mapper.resource("flavor", "flavors", - controller=controllers.FlavorsController()) - mapper.resource("sharedipgroup", "sharedipgroups", - controller=controllers.SharedIpGroupsController()) - - super(APIRouter, self).__init__(mapper) diff --git a/nova/endpoint/rackspace/controllers/__init__.py b/nova/endpoint/rackspace/controllers/__init__.py deleted file mode 100644 index 052b6f36..00000000 --- a/nova/endpoint/rackspace/controllers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from nova.endpoint.rackspace.controllers.images import ImagesController -from nova.endpoint.rackspace.controllers.flavors import FlavorsController -from nova.endpoint.rackspace.controllers.servers import ServersController -from nova.endpoint.rackspace.controllers.sharedipgroups import \ - SharedIpGroupsController diff --git a/nova/endpoint/rackspace/controllers/base.py b/nova/endpoint/rackspace/controllers/base.py deleted file mode 100644 index 8cd44f62..00000000 --- a/nova/endpoint/rackspace/controllers/base.py +++ /dev/null @@ -1,9 +0,0 @@ -from nova import wsgi - -class BaseController(wsgi.Controller): - @classmethod - def render(cls, instance): - if isinstance(instance, list): - return { cls.entity_name : cls.render(instance) } - else: - return { "TODO": "TODO" } diff --git a/nova/endpoint/rackspace/controllers/flavors.py b/nova/endpoint/rackspace/controllers/flavors.py deleted file mode 100644 index f256cc85..00000000 --- a/nova/endpoint/rackspace/controllers/flavors.py +++ /dev/null @@ -1 +0,0 @@ -class FlavorsController(object): pass diff --git a/nova/endpoint/rackspace/controllers/images.py b/nova/endpoint/rackspace/controllers/images.py deleted file mode 100644 index ae2a0884..00000000 --- a/nova/endpoint/rackspace/controllers/images.py +++ /dev/null @@ -1 +0,0 @@ -class ImagesController(object): pass diff --git a/nova/endpoint/rackspace/controllers/servers.py b/nova/endpoint/rackspace/controllers/servers.py deleted file mode 100644 index 2f8e662d..00000000 --- a/nova/endpoint/rackspace/controllers/servers.py +++ /dev/null @@ -1,63 +0,0 @@ -from nova import rpc -from nova.compute import model as compute -from nova.endpoint.rackspace.controllers.base import BaseController - -class ServersController(BaseController): - entity_name = 'servers' - - def index(self, **kwargs): - return [instance_details(inst) for inst in compute.InstanceDirectory().all] - - def show(self, **kwargs): - instance_id = kwargs['id'] - return compute.InstanceDirectory().get(instance_id) - - def delete(self, **kwargs): - instance_id = kwargs['id'] - instance = compute.InstanceDirectory().get(instance_id) - if not instance: - raise ServerNotFound("The requested server was not found") - instance.destroy() - return True - - def create(self, **kwargs): - inst = self.build_server_instance(kwargs['server']) - rpc.cast( - FLAGS.compute_topic, { - "method": "run_instance", - "args": {"instance_id": inst.instance_id}}) - - def update(self, **kwargs): - instance_id = kwargs['id'] - instance = compute.InstanceDirectory().get(instance_id) - if not instance: - raise ServerNotFound("The requested server was not found") - instance.update(kwargs['server']) - instance.save() - - def build_server_instance(self, env): - """Build instance data structure and save it to the data store.""" - reservation = utils.generate_uid('r') - ltime = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) - inst = self.instdir.new() - inst['name'] = env['server']['name'] - inst['image_id'] = env['server']['imageId'] - inst['instance_type'] = env['server']['flavorId'] - inst['user_id'] = env['user']['id'] - inst['project_id'] = env['project']['id'] - inst['reservation_id'] = reservation - inst['launch_time'] = ltime - inst['mac_address'] = utils.generate_mac() - address = self.network.allocate_ip( - inst['user_id'], - inst['project_id'], - mac=inst['mac_address']) - inst['private_dns_name'] = str(address) - inst['bridge_name'] = network.BridgedNetwork.get_network_for_project( - inst['user_id'], - inst['project_id'], - 'default')['bridge_name'] - # key_data, key_name, ami_launch_index - # TODO(todd): key data or root password - inst.save() - return inst diff --git a/nova/endpoint/rackspace/controllers/sharedipgroups.py b/nova/endpoint/rackspace/controllers/sharedipgroups.py deleted file mode 100644 index 9d346d62..00000000 --- a/nova/endpoint/rackspace/controllers/sharedipgroups.py +++ /dev/null @@ -1 +0,0 @@ -class SharedIpGroupsController(object): pass From 274241734b92d417ef77df6fbb49e74c9aa887b3 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 18 Aug 2010 10:39:59 -0400 Subject: [PATCH 63/64] Newest pylint supports 'disable=', not 'disable-msg=' --- bin/nova-rsapi | 2 +- nova/tests/objectstore_unittest.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bin/nova-rsapi b/bin/nova-rsapi index e2722422..9ad6f9e9 100755 --- a/bin/nova-rsapi +++ b/bin/nova-rsapi @@ -1,5 +1,5 @@ #!/usr/bin/env python -# pylint: disable-msg=C0103 +# pylint: disable=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index dece4b5d..5b956fcc 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -56,7 +56,7 @@ os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets')) class ObjectStoreTestCase(test.BaseTestCase): """Test objectstore API directly.""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): # pylint: disable=C0103 """Setup users and projects.""" super(ObjectStoreTestCase, self).setUp() self.flags(buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'), @@ -78,7 +78,7 @@ class ObjectStoreTestCase(test.BaseTestCase): self.context = Context() - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): # pylint: disable=C0103 """Tear down users and projects.""" self.auth_manager.delete_project('proj1') self.auth_manager.delete_project('proj2') @@ -168,7 +168,7 @@ class ObjectStoreTestCase(test.BaseTestCase): class TestHTTPChannel(http.HTTPChannel): """Dummy site required for twisted.web""" - def checkPersistence(self, _, __): # pylint: disable-msg=C0103 + def checkPersistence(self, _, __): # pylint: disable=C0103 """Otherwise we end up with an unclean reactor.""" return False @@ -181,7 +181,7 @@ class TestSite(server.Site): class S3APITestCase(test.TrialTestCase): """Test objectstore through S3 API.""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): # pylint: disable=C0103 """Setup users, projects, and start a test server.""" super(S3APITestCase, self).setUp() @@ -198,7 +198,7 @@ class S3APITestCase(test.TrialTestCase): root = S3() self.site = TestSite(root) - # pylint: disable-msg=E1101 + # pylint: disable=E1101 self.listening_port = reactor.listenTCP(0, self.site, interface='127.0.0.1') # pylint: enable-msg=E1101 @@ -221,11 +221,11 @@ class S3APITestCase(test.TrialTestCase): self.conn.get_http_connection = get_http_connection - def _ensure_no_buckets(self, buckets): # pylint: disable-msg=C0111 + def _ensure_no_buckets(self, buckets): # pylint: disable=C0111 self.assertEquals(len(buckets), 0, "Bucket list was not empty") return True - def _ensure_one_bucket(self, buckets, name): # pylint: disable-msg=C0111 + def _ensure_one_bucket(self, buckets, name): # pylint: disable=C0111 self.assertEquals(len(buckets), 1, "Bucket list didn't have exactly one element in it") self.assertEquals(buckets[0].name, name, "Wrong name") @@ -296,7 +296,7 @@ class S3APITestCase(test.TrialTestCase): deferred.addCallback(self._ensure_no_buckets) return deferred - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): # pylint: disable=C0103 """Tear down auth and test server.""" self.auth_manager.delete_user('admin') self.auth_manager.delete_project('admin') From 79322e654310c7ccd40dfa3b95a8e6253ebea1e5 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 18 Aug 2010 11:44:24 -0400 Subject: [PATCH 64/64] Changed our minds: keep pylint equal to Ubuntu Lucid version, and use disable-msg throughout. --- bin/nova-rsapi | 2 +- nova/rpc.py | 8 ++++---- nova/tests/network_unittest.py | 4 ++-- nova/tests/objectstore_unittest.py | 16 ++++++++-------- nova/tests/rpc_unittest.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bin/nova-rsapi b/bin/nova-rsapi index 9ad6f9e9..e2722422 100755 --- a/bin/nova-rsapi +++ b/bin/nova-rsapi @@ -1,5 +1,5 @@ #!/usr/bin/env python -# pylint: disable=C0103 +# pylint: disable-msg=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the diff --git a/nova/rpc.py b/nova/rpc.py index 824a66b5..84a9b559 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -59,7 +59,7 @@ class Connection(carrot_connection.BrokerConnection): params['backend_cls'] = fakerabbit.Backend # NOTE(vish): magic is fun! - # pylint: disable=W0142 + # pylint: disable-msg=W0142 cls._instance = cls(**params) return cls._instance @@ -104,7 +104,7 @@ class Consumer(messaging.Consumer): if self.failed_connection: # NOTE(vish): conn is defined in the parent class, we can # recreate it as long as we create the backend too - # pylint: disable=W0201 + # pylint: disable-msg=W0201 self.conn = Connection.recreate() self.backend = self.conn.create_backend() super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) @@ -114,7 +114,7 @@ class Consumer(messaging.Consumer): # NOTE(vish): This is catching all errors because we really don't # exceptions to be logged 10 times a second if some # persistent failure occurs. - except Exception: # pylint: disable=W0703 + except Exception: # pylint: disable-msg=W0703 if not self.failed_connection: logging.exception("Failed to fetch message from queue") self.failed_connection = True @@ -178,7 +178,7 @@ class AdapterConsumer(TopicConsumer): node_func = getattr(self.proxy, str(method)) node_args = dict((str(k), v) for k, v in args.iteritems()) # NOTE(vish): magic is fun! - # pylint: disable=W0142 + # pylint: disable-msg=W0142 d = defer.maybeDeferred(node_func, **node_args) if msg_id: d.addCallback(lambda rval: msg_reply(msg_id, rval, None)) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 03950980..993bfacc 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -36,7 +36,7 @@ FLAGS = flags.FLAGS class NetworkTestCase(test.TrialTestCase): """Test cases for network code""" - def setUp(self): # pylint: disable=C0103 + def setUp(self): # pylint: disable-msg=C0103 super(NetworkTestCase, self).setUp() # NOTE(vish): if you change these flags, make sure to change the # flags in the corresponding section in nova-dhcpbridge @@ -60,7 +60,7 @@ class NetworkTestCase(test.TrialTestCase): vpn.NetworkData.create(self.projects[i].id) self.service = service.VlanNetworkService() - def tearDown(self): # pylint: disable=C0103 + def tearDown(self): # pylint: disable-msg=C0103 super(NetworkTestCase, self).tearDown() for project in self.projects: self.manager.delete_project(project) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index 5b956fcc..dece4b5d 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -56,7 +56,7 @@ os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets')) class ObjectStoreTestCase(test.BaseTestCase): """Test objectstore API directly.""" - def setUp(self): # pylint: disable=C0103 + def setUp(self): # pylint: disable-msg=C0103 """Setup users and projects.""" super(ObjectStoreTestCase, self).setUp() self.flags(buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'), @@ -78,7 +78,7 @@ class ObjectStoreTestCase(test.BaseTestCase): self.context = Context() - def tearDown(self): # pylint: disable=C0103 + def tearDown(self): # pylint: disable-msg=C0103 """Tear down users and projects.""" self.auth_manager.delete_project('proj1') self.auth_manager.delete_project('proj2') @@ -168,7 +168,7 @@ class ObjectStoreTestCase(test.BaseTestCase): class TestHTTPChannel(http.HTTPChannel): """Dummy site required for twisted.web""" - def checkPersistence(self, _, __): # pylint: disable=C0103 + def checkPersistence(self, _, __): # pylint: disable-msg=C0103 """Otherwise we end up with an unclean reactor.""" return False @@ -181,7 +181,7 @@ class TestSite(server.Site): class S3APITestCase(test.TrialTestCase): """Test objectstore through S3 API.""" - def setUp(self): # pylint: disable=C0103 + def setUp(self): # pylint: disable-msg=C0103 """Setup users, projects, and start a test server.""" super(S3APITestCase, self).setUp() @@ -198,7 +198,7 @@ class S3APITestCase(test.TrialTestCase): root = S3() self.site = TestSite(root) - # pylint: disable=E1101 + # pylint: disable-msg=E1101 self.listening_port = reactor.listenTCP(0, self.site, interface='127.0.0.1') # pylint: enable-msg=E1101 @@ -221,11 +221,11 @@ class S3APITestCase(test.TrialTestCase): self.conn.get_http_connection = get_http_connection - def _ensure_no_buckets(self, buckets): # pylint: disable=C0111 + def _ensure_no_buckets(self, buckets): # pylint: disable-msg=C0111 self.assertEquals(len(buckets), 0, "Bucket list was not empty") return True - def _ensure_one_bucket(self, buckets, name): # pylint: disable=C0111 + def _ensure_one_bucket(self, buckets, name): # pylint: disable-msg=C0111 self.assertEquals(len(buckets), 1, "Bucket list didn't have exactly one element in it") self.assertEquals(buckets[0].name, name, "Wrong name") @@ -296,7 +296,7 @@ class S3APITestCase(test.TrialTestCase): deferred.addCallback(self._ensure_no_buckets) return deferred - def tearDown(self): # pylint: disable=C0103 + def tearDown(self): # pylint: disable-msg=C0103 """Tear down auth and test server.""" self.auth_manager.delete_user('admin') self.auth_manager.delete_project('admin') diff --git a/nova/tests/rpc_unittest.py b/nova/tests/rpc_unittest.py index 764a9741..e12a28fb 100644 --- a/nova/tests/rpc_unittest.py +++ b/nova/tests/rpc_unittest.py @@ -32,7 +32,7 @@ FLAGS = flags.FLAGS class RpcTestCase(test.BaseTestCase): """Test cases for rpc""" - def setUp(self): # pylint: disable=C0103 + def setUp(self): # pylint: disable-msg=C0103 super(RpcTestCase, self).setUp() self.conn = rpc.Connection.instance() self.receiver = TestReceiver()