From 4cffebb5687198ccbac2b88ec332f992025478cf Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:28:04 -0500 Subject: [PATCH 01/24] execute: shell=True removed. --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..40a8d8d8c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,7 +125,7 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl --fail %s -o %s" % (url, target)) + execute("curl","--fail",url,"-o",target) def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): @@ -133,7 +133,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip -f inet6 -o addr show %s" % interface) + if_str = execute("ip","-f","inet6","-o","addr","show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] From 78912390cefc2dfaa1355591d05732f2b4787230 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 12:37:02 +0100 Subject: [PATCH 02/24] Add utils.synchronized decorator to allow for synchronising method entrance across multiple workers on the same host. --- nova/tests/test_misc.py | 37 ++++++++++++++++++++++++++++++++++++- nova/utils.py | 11 +++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index e6da6112a..154b6fae6 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -14,10 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. +from datetime import datetime +import errno import os +import select +import time from nova import test -from nova.utils import parse_mailmap, str_dict_replace +from nova.utils import parse_mailmap, str_dict_replace, synchronized class ProjectTestCase(test.TestCase): @@ -55,3 +59,34 @@ class ProjectTestCase(test.TestCase): '%r not listed in Authors' % missing) finally: tree.unlock() + + +class LockTestCase(test.TestCase): + def test_synchronized(self): + rpipe, wpipe = os.pipe() + pid = os.fork() + if pid > 0: + os.close(wpipe) + + @synchronized('testlock') + def f(): + rfds, _, __ = select.select([rpipe], [], [], 1) + self.assertEquals(len(rfds), 0, "The other process, which was" + " supposed to be locked, " + "wrote on its end of the " + "pipe") + os.close(rpipe) + + f() + else: + os.close(rpipe) + + @synchronized('testlock') + def g(): + try: + os.write(wpipe, "foo") + except OSError, e: + self.assertEquals(e.errno, errno.EPIPE) + return + g() + os._exit(0) diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..cb1ea5a7d 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -25,6 +25,7 @@ import base64 import datetime import inspect import json +import lockfile import os import random import socket @@ -491,6 +492,16 @@ def loads(s): return json.loads(s) +def synchronized(name): + def wrap(f): + def inner(*args, **kwargs): + lock = lockfile.FileLock('nova-%s.lock' % name) + with lock: + return f(*args, **kwargs) + return inner + return wrap + + def ensure_b64_encoding(val): """Safety method to ensure that values expected to be base64-encoded actually are. If they are, the value is returned unchanged. Otherwise, From 22d504b719e68a0f7a78a1ddad6811498682be4d Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 23:31:09 +0100 Subject: [PATCH 03/24] Add a lock_path flag for lock files. --- nova/flags.py | 2 ++ nova/utils.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index 8cf199b2f..213d4d4e1 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -321,6 +321,8 @@ DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), "Top-level directory for maintaining nova's state") +DEFINE_string('lock_path', os.path.join(os.path.dirname(__file__), '../'), + "Directory for lock files") DEFINE_string('logdir', None, 'output to a per-service log file in named ' 'directory') diff --git a/nova/utils.py b/nova/utils.py index cb1ea5a7d..48f12350f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -44,11 +44,13 @@ from eventlet.green import subprocess from nova import exception from nova.exception import ProcessExecutionError +from nova import flags from nova import log as logging LOG = logging.getLogger("nova.utils") TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +FLAGS = flags.FLAGS def import_class(import_str): @@ -495,7 +497,8 @@ def loads(s): def synchronized(name): def wrap(f): def inner(*args, **kwargs): - lock = lockfile.FileLock('nova-%s.lock' % name) + lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, + 'nova-%s.lock' % name)) with lock: return f(*args, **kwargs) return inner From f74a5b7c8abd83401d1457fc1bb1ba20cd9e8a37 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 1 Mar 2011 20:49:46 +0100 Subject: [PATCH 04/24] Use functools.wraps to make sure wrapped method's metadata (docstring and name) doesn't get mangled. --- nova/tests/test_misc.py | 12 ++++++++++-- nova/utils.py | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 154b6fae6..9f572b58e 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -14,11 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -from datetime import datetime import errno import os import select -import time from nova import test from nova.utils import parse_mailmap, str_dict_replace, synchronized @@ -62,6 +60,16 @@ class ProjectTestCase(test.TestCase): class LockTestCase(test.TestCase): + def test_synchronized_wrapped_function_metadata(self): + @synchronized('whatever') + def foo(): + """Bar""" + pass + self.assertEquals(foo.__doc__, 'Bar', "Wrapped function's docstring " + "got lost") + self.assertEquals(foo.__name__, 'foo', "Wrapped function's name " + "got mangled") + def test_synchronized(self): rpipe, wpipe = os.pipe() pid = os.fork() diff --git a/nova/utils.py b/nova/utils.py index 48f12350f..9929e6fef 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -23,11 +23,14 @@ System-level utilities and helper functions. import base64 import datetime +import functools import inspect import json import lockfile +import netaddr import os import random +import re import socket import string import struct @@ -35,8 +38,6 @@ import sys import time import types from xml.sax import saxutils -import re -import netaddr from eventlet import event from eventlet import greenthread @@ -496,6 +497,7 @@ def loads(s): def synchronized(name): def wrap(f): + @functools.wraps(f) def inner(*args, **kwargs): lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, 'nova-%s.lock' % name)) From a748092bcc618fbbdb9d593492e64285080e63c1 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 11:04:33 -0500 Subject: [PATCH 05/24] adding wsgi.Request class to add custom best_match; adding new class to wsgify decorators; replacing all references to webob.Request in non-test code to wsgi.Request --- nova/wsgi.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index 1eb66d067..67216d540 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -82,6 +82,27 @@ class Server(object): log=WritableLogger(logger)) +class Request(webob.Request): + + def best_match(self): + """ + Determine the most acceptable content-type based on the + query extension then the Accept header + """ + + parts = self.path.rsplit(".", 1) + + if len(parts) > 1: + format = parts[1] + if format in ["json", "xml"]: + return "application/{0}".format(parts[1]) + + ctypes = ["application/json", "application/xml"] + bm = self.accept.best_match(ctypes) + + return bm or "application/json" + + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -113,7 +134,7 @@ class Application(object): def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): # Any of the following objects work as responses: @@ -199,7 +220,7 @@ class Middleware(Application): """Do whatever you'd like to the response.""" return response - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): response = self.process_request(req) if response: @@ -212,7 +233,7 @@ class Debug(Middleware): """Helper class that can be inserted into any WSGI application chain to get information about the request and response.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): print ("*" * 40) + " REQUEST ENVIRON" for key, value in req.environ.items(): @@ -276,7 +297,7 @@ class Router(object): self._router = routes.middleware.RoutesMiddleware(self._dispatch, self.map) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Route the incoming request to a controller based on self.map. @@ -285,7 +306,7 @@ class Router(object): return self._router @staticmethod - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def _dispatch(req): """ Called by self._router after matching the incoming request to a route @@ -304,11 +325,11 @@ class Controller(object): WSGI app that reads routing information supplied by RoutesMiddleware and calls the requested action method upon itself. All action methods must, in addition to their normal parameters, accept a 'req' argument - which is the incoming webob.Request. They raise a webob.exc exception, + which is the incoming wsgi.Request. They raise a webob.exc exception, or return a dict which will be serialized by requested content type. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Call the method specified in req.environ by RoutesMiddleware. @@ -358,7 +379,7 @@ class Serializer(object): needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = webob.Request.blank('', environ) + req = wsgi.Request.blank('', environ) suffix = req.path_info.split('.')[-1].lower() if suffix == 'json': self.handler = self._to_json From 44e4bebb09e711eee8fcbaa0a6684360999cca87 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 12:45:34 -0500 Subject: [PATCH 06/24] adding wsgi.Controller and wsgi.Request testing; fixing format keyword argument exception --- nova/wsgi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index 67216d540..4577439cb 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -339,6 +339,8 @@ class Controller(object): method = getattr(self, action) del arg_dict['controller'] del arg_dict['action'] + if 'format' in arg_dict: + del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) if type(result) is dict: @@ -379,7 +381,7 @@ class Serializer(object): needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = wsgi.Request.blank('', environ) + req = Request.blank('', environ) suffix = req.path_info.split('.')[-1].lower() if suffix == 'json': self.handler = self._to_json From 759c0e546f63e4d95df31a9ad904853205dcc44c Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 17:21:21 -0500 Subject: [PATCH 07/24] Refactor wsgi.Serializer away from handling Requests directly; now require Content-Type in all requests; fix tests according to new code --- nova/exception.py | 4 ++ nova/tests/test_direct.py | 3 ++ nova/wsgi.py | 96 ++++++++++++++++++++++++--------------- 3 files changed, 66 insertions(+), 37 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index 7d65bd6a5..93c5fe3d7 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -88,6 +88,10 @@ class InvalidInputException(Error): pass +class InvalidContentType(Error): + pass + + class TimeoutException(Error): pass diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index b6bfab534..85bfcfd85 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -59,6 +59,7 @@ class DirectTestCase(test.TestCase): req.headers['X-OpenStack-User'] = 'user1' req.headers['X-OpenStack-Project'] = 'proj1' resp = req.get_response(self.auth_router) + self.assertEqual(resp.status_int, 200) data = json.loads(resp.body) self.assertEqual(data['user'], 'user1') self.assertEqual(data['project'], 'proj1') @@ -69,6 +70,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'json=%s' % json.dumps({'data': 'foo'}) resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') @@ -78,6 +80,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'data=foo' resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') diff --git a/nova/wsgi.py b/nova/wsgi.py index 4577439cb..c3e08522d 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -36,6 +36,7 @@ import webob.exc from paste import deploy +from nova import exception from nova import flags from nova import log as logging from nova import utils @@ -102,6 +103,14 @@ class Request(webob.Request): return bm or "application/json" + def get_content_type(self): + try: + ct = self.headers["Content-Type"] + assert ct in ("application/xml", "application/json") + return ct + except Exception: + raise webob.exc.HTTPBadRequest("Invalid content type") + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -343,30 +352,41 @@ class Controller(object): del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) + if type(result) is dict: - return self._serialize(result, req) + content_type = req.best_match() + body = self._serialize(result, content_type) + + response = webob.Response() + response.headers["Content-Type"] = content_type + response.body = body + return response + else: return result - def _serialize(self, data, request): + def _serialize(self, data, content_type): """ - Serialize the given dict to the response type requested in request. + Serialize the given dict to the provided content_type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.to_content_type(data) + serializer = Serializer(_metadata) + try: + return serializer.serialize(data, content_type) + except exception.InvalidContentType: + raise webob.exc.HTTPNotAcceptable() - def _deserialize(self, data, request): + def _deserialize(self, data, content_type): """ - Deserialize the request body to the response type requested in request. + Deserialize the request body to the specefied content type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.deserialize(data) + serializer = Serializer(_metadata) + return serializer.deserialize(data, content_type) class Serializer(object): @@ -374,50 +394,52 @@ class Serializer(object): Serializes and deserializes dictionaries to certain MIME types. """ - def __init__(self, environ, metadata=None): + def __init__(self, metadata=None): """ Create a serializer based on the given WSGI environment. 'metadata' is an optional dict mapping MIME types to information needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = Request.blank('', environ) - suffix = req.path_info.split('.')[-1].lower() - if suffix == 'json': - self.handler = self._to_json - elif suffix == 'xml': - self.handler = self._to_xml - elif 'application/json' in req.accept: - self.handler = self._to_json - elif 'application/xml' in req.accept: - self.handler = self._to_xml - else: - # This is the default - self.handler = self._to_json - def to_content_type(self, data): + def _get_serialize_handler(self, content_type): + handlers = { + "application/json": self._to_json, + "application/xml": self._to_xml, + } + + try: + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() + + def serialize(self, data, content_type): """ - Serialize a dictionary into a string. - - The format of the string will be decided based on the Content Type - requested in self.environ: by Accept: header, or by URL suffix. + Serialize a dictionary into a string of the specified content type. """ - return self.handler(data) + return self._get_serialize_handler(content_type)(data) - def deserialize(self, datastring): + def deserialize(self, datastring, content_type): """ Deserialize a string to a dictionary. The string must be in the format of a supported MIME type. """ - datastring = datastring.strip() + return self.get_deserialize_handler(content_type)(datastring) + + def get_deserialize_handler(self, content_type): + handlers = { + "application/json": self._from_json, + "application/xml": self._from_xml, + } + try: - is_xml = (datastring[0] == '<') - if not is_xml: - return utils.loads(datastring) - return self._from_xml(datastring) - except: - return None + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() + + def _from_json(self, datastring): + return utils.loads(datastring) def _from_xml(self, datastring): xmldata = self.metadata.get('application/xml', {}) From a2c760844da82b36b109d3a6a391c987d2df80b7 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Fri, 4 Mar 2011 21:07:03 +0100 Subject: [PATCH 08/24] Added initial support to delete networks nova-manage --- bin/nova-manage | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/nova-manage b/bin/nova-manage index 9bf3a1bb3..9557f2423 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -546,6 +546,16 @@ class NetworkCommands(object): network.dns) + def delete(self, fixed_range): + """Deletes a network""" + try: + network = [n for n in db.network_get_all(context.get_admin_context()) + if n.cidr == fixed_range][0] + + print network.id, network.cidr, network.project_id + except IndexError: + raise ValueError(_("Network does not exist")) + class ServiceCommands(object): """Enable and disable running services""" From 3f18175cb681de74391de26a17cccdbaf025ae43 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 7 Mar 2011 22:18:15 +0100 Subject: [PATCH 09/24] added network_get_by_cidr method to nova.db api --- bin/nova-manage | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 9557f2423..b274c5bd1 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -545,16 +545,9 @@ class NetworkCommands(object): network.dhcp_start, network.dns) - def delete(self, fixed_range): """Deletes a network""" - try: - network = [n for n in db.network_get_all(context.get_admin_context()) - if n.cidr == fixed_range][0] - - print network.id, network.cidr, network.project_id - except IndexError: - raise ValueError(_("Network does not exist")) + print db.network_get_by_cidr(context.get_admin_context(), fixed_range) class ServiceCommands(object): """Enable and disable running services""" From 362d39ddfd6072893b64008dcfca9c2b9946ff7e Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 7 Mar 2011 22:37:26 +0100 Subject: [PATCH 10/24] deleted network_is_associated from nova.db api --- bin/nova-manage | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b274c5bd1..94b0d5946 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -547,8 +547,8 @@ class NetworkCommands(object): def delete(self, fixed_range): """Deletes a network""" - print db.network_get_by_cidr(context.get_admin_context(), fixed_range) - + network = db.network_get_by_cidr(context.get_admin_context(), fixed_range) + class ServiceCommands(object): """Enable and disable running services""" From e864b0cc441351c39dbdca4ecd5b7918408d658f Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Tue, 8 Mar 2011 01:01:41 -0500 Subject: [PATCH 11/24] execvp --- nova/tests/test_network.py | 16 ++++++++-------- nova/utils.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index ce1c77210..6d2d8b771 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -343,13 +343,13 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s add %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'add' + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("ISSUE_IP: %s, %s ", out, err) @@ -359,11 +359,11 @@ def release_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s del %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'del', + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("RELEASE_IP: %s, %s ", out, err) diff --git a/nova/utils.py b/nova/utils.py index 40a8d8d8c..c96b85294 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,15 +125,15 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl","--fail",url,"-o",target) + execute("curl", "--fail", url, "-o", target) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) +def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, + obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -148,7 +148,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) # NOTE(termie): this appears to be necessary to let the subprocess call # clean something up in between calls, without it two # execute calls in a row hangs the second one @@ -158,7 +158,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): def ssh_execute(ssh, cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (SSH): %s"), cmd) + LOG.debug(_("Running cmd (SSH): %s"), ' '.join(cmd)) if addl_env: raise exception.Error("Environment not supported over SSH") @@ -187,7 +187,7 @@ def ssh_execute(ssh, cmd, process_input=None, raise exception.ProcessExecutionError(exit_code=exit_status, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) return (stdout, stderr) @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip","-f","inet6","-o","addr","show", interface) + if_str = execute("ip", "-f", "inet6", "-o", "addr", "show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] From 400658afaf584dd22391636e9f24e49daf6d86c2 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Tue, 8 Mar 2011 18:53:20 +0100 Subject: [PATCH 12/24] Added ability to remove networks on nova-manage command --- bin/nova-manage | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 94b0d5946..8ddfea5c2 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -548,7 +548,11 @@ class NetworkCommands(object): def delete(self, fixed_range): """Deletes a network""" network = db.network_get_by_cidr(context.get_admin_context(), fixed_range) - + if network.project_id is not None: + raise ValueError(_('Network must be disassociated from project %s' + ' before delete' %network.project_id)) + db.network_delete_safe(context.get_admin_context(), network.id) + class ServiceCommands(object): """Enable and disable running services""" From 981643d670da324d99b4239cb8397f09cf9b569a Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 00:30:05 -0500 Subject: [PATCH 13/24] execvp: almost passes tests --- nova/tests/test_network.py | 2 +- nova/tests/test_virt.py | 11 ++++++----- nova/utils.py | 19 +++++++++++++------ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 6d2d8b771..19099ff4c 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -343,7 +343,7 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = (binpath('nova-dhcpbridge'), 'add' + cmd = (binpath('nova-dhcpbridge'), 'add', instance_ref['mac_address'], private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f151ae911..7f1ad002e 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -315,15 +315,16 @@ class IptablesFirewallTestCase(test.TestCase): instance_ref = db.instance_get(admin_ctxt, instance_ref['id']) # self.fw.add_instance(instance_ref) - def fake_iptables_execute(cmd, process_input=None): - if cmd == 'sudo ip6tables-save -t filter': + def fake_iptables_execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): return '\n'.join(self.in6_rules), None - if cmd == 'sudo iptables-save -t filter': + if cmd == ('sudo', 'iptables-save', '-t', 'filter'): return '\n'.join(self.in_rules), None - if cmd == 'sudo iptables-restore': + if cmd == ('sudo', 'iptables-restore'): self.out_rules = process_input.split('\n') return '', '' - if cmd == 'sudo ip6tables-restore': + if cmd == ('sudo', 'ip6tables-restore'): self.out6_rules = process_input.split('\n') return '', '' self.fw.execute = fake_iptables_execute diff --git a/nova/utils.py b/nova/utils.py index c96b85294..9b51f8b40 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -128,13 +128,20 @@ def fetchfile(url, target): execute("curl", "--fail", url, "-o", target) -def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): +def execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + addl_env=kwargs.get('addl_env', None) + check_exit_code=kwargs.get('check_exit_code', True) + stdin=kwargs.get('stdin', subprocess.PIPE) + stdout=kwargs.get('stdout', subprocess.PIPE) + stderr=kwargs.get('stderr', subprocess.PIPE) + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + obj = subprocess.Popen(cmd, stdin=stdin, + stdout=stdout, stderr=stderr, env=env) result = None if process_input != None: result = obj.communicate(process_input) @@ -220,9 +227,9 @@ def debug(arg): return arg -def runthis(prompt, cmd, check_exit_code=True): - LOG.debug(_("Running %s"), (cmd)) - rv, err = execute(cmd, check_exit_code=check_exit_code) +def runthis(prompt, *cmd, **kwargs): + LOG.debug(_("Running %s"), (" ".join(cmd))) + rv, err = execute(*cmd, **kwargs) def generate_uid(topic, size=8): From 59bfead676528ce8ea81c13f755cec81c7287fae Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 01:26:53 -0500 Subject: [PATCH 14/24] execvp: unit tests pass --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/utils.py b/nova/utils.py index 9b51f8b40..7ddf056ea 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -135,6 +135,7 @@ def execute(*cmd, **kwargs): stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) + cmd=map(str,cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() From 5a5c7e32049409d492b93bbb49b3361aff41d731 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 06:56:42 +0000 Subject: [PATCH 15/24] tests and semaphore fix for image caching --- nova/tests/test_virt.py | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f151ae911..ec7498d72 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,12 +14,16 @@ # License for the specific language governing permissions and limitations # under the License. +import os + +import eventlet from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db from nova import flags +from nova import log as logging from nova import test from nova import utils from nova.api.ec2 import cloud @@ -30,6 +34,68 @@ FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +def _concurrency(wait, done, target): + wait.wait() + done.send() + + +class CacheConcurrencyTestCase(test.TestCase): + def setUp(self): + super(CacheConcurrencyTestCase, self).setUp() + + def fake_exists(fname): + basedir = os.path.join(FLAGS.instances_path, '_base') + if fname == basedir: + return True + return False + + def fake_execute(*args, **kwargs): + pass + + self.stubs.Set(os.path, 'exists', fake_exists) + self.stubs.Set(utils, 'execute', fake_execute) + + def test_same_fname_concurrency(self): + """Ensures that the same fname cache runs at a sequentially""" + conn = libvirt_conn.get_connection(False) + wait1 = eventlet.event.Event() + done1 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname', False, wait1, done1) + wait2 = eventlet.event.Event() + done2 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname', False, wait2, done2) + wait2.send() + eventlet.sleep(0) + try: + self.assertFalse(done2.ready()) + finally: + wait1.send() + done1.wait() + eventlet.sleep(0) + self.assertTrue(done2.ready()) + + def test_different_fname_concurrency(self): + """Ensures that two different fname caches are concurrent""" + conn = libvirt_conn.get_connection(False) + wait1 = eventlet.event.Event() + done1 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname2', False, wait1, done1) + wait2 = eventlet.event.Event() + done2 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname1', False, wait2, done2) + wait2.send() + eventlet.sleep(0) + try: + self.assertTrue(done2.ready()) + finally: + wait1.send() + eventlet.sleep(0) + + class LibvirtConnTestCase(test.TestCase): def setUp(self): super(LibvirtConnTestCase, self).setUp() From d08c12fc39a62e50468d48639d3d6c527a887246 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 07:35:58 +0000 Subject: [PATCH 16/24] make static method for testing without initializing libvirt --- nova/tests/test_virt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index ec7498d72..113632a0c 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -57,7 +57,7 @@ class CacheConcurrencyTestCase(test.TestCase): def test_same_fname_concurrency(self): """Ensures that the same fname cache runs at a sequentially""" - conn = libvirt_conn.get_connection(False) + conn = libvirt_conn.LibvirtConnection wait1 = eventlet.event.Event() done1 = eventlet.event.Event() eventlet.spawn(conn._cache_image, _concurrency, @@ -78,7 +78,7 @@ class CacheConcurrencyTestCase(test.TestCase): def test_different_fname_concurrency(self): """Ensures that two different fname caches are concurrent""" - conn = libvirt_conn.get_connection(False) + conn = libvirt_conn.LibvirtConnection wait1 = eventlet.event.Event() done1 = eventlet.event.Event() eventlet.spawn(conn._cache_image, _concurrency, From a81328b0e870e1155a3c2a680541f94bdb981a43 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 9 Mar 2011 10:30:18 +0100 Subject: [PATCH 17/24] Stop assuming anything about the order in which the two processes are scheduled. --- nova/tests/test_misc.py | 47 ++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 9f572b58e..a658e4978 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -71,30 +71,33 @@ class LockTestCase(test.TestCase): "got mangled") def test_synchronized(self): - rpipe, wpipe = os.pipe() - pid = os.fork() - if pid > 0: - os.close(wpipe) + rpipe1, wpipe1 = os.pipe() + rpipe2, wpipe2 = os.pipe() - @synchronized('testlock') - def f(): - rfds, _, __ = select.select([rpipe], [], [], 1) - self.assertEquals(len(rfds), 0, "The other process, which was" - " supposed to be locked, " - "wrote on its end of the " - "pipe") - os.close(rpipe) + @synchronized('testlock') + def f(rpipe, wpipe): + try: + os.write(wpipe, "foo") + except OSError, e: + self.assertEquals(e.errno, errno.EPIPE) + return - f() - else: + rfds, _, __ = select.select([rpipe], [], [], 1) + self.assertEquals(len(rfds), 0, "The other process, which was" + " supposed to be locked, " + "wrote on its end of the " + "pipe") os.close(rpipe) - @synchronized('testlock') - def g(): - try: - os.write(wpipe, "foo") - except OSError, e: - self.assertEquals(e.errno, errno.EPIPE) - return - g() + pid = os.fork() + if pid > 0: + os.close(wpipe1) + os.close(rpipe2) + + f(rpipe1, wpipe2) + else: + os.close(rpipe1) + os.close(wpipe2) + + f(rpipe2, wpipe1) os._exit(0) From ee48d77428976755829ac8efbdc29e0f00735b7b Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 9 Mar 2011 19:03:58 +0100 Subject: [PATCH 18/24] fixed lp715427 --- bin/nova-manage | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 8ddfea5c2..45437d7e7 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -547,10 +547,11 @@ class NetworkCommands(object): def delete(self, fixed_range): """Deletes a network""" - network = db.network_get_by_cidr(context.get_admin_context(), fixed_range) + network = db.network_get_by_cidr(context.get_admin_context(), \ + fixed_range) if network.project_id is not None: - raise ValueError(_('Network must be disassociated from project %s' - ' before delete' %network.project_id)) + raise ValueError(_('Network must be disassociated from project %s' + ' before delete' % network.project_id)) db.network_delete_safe(context.get_admin_context(), network.id) class ServiceCommands(object): From f431c0f8846f52070779a89b26d0a67d6d5bbb0f Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 9 Mar 2011 19:16:26 +0100 Subject: [PATCH 19/24] Fixed pep8 issues --- bin/nova-manage | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 45437d7e7..7dfc3c045 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -552,8 +552,9 @@ class NetworkCommands(object): if network.project_id is not None: raise ValueError(_('Network must be disassociated from project %s' ' before delete' % network.project_id)) - db.network_delete_safe(context.get_admin_context(), network.id) - + db.network_delete_safe(context.get_admin_context(), network.id) + + class ServiceCommands(object): """Enable and disable running services""" From 866048da048c5037f7d8ce0331778327b0318888 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 14:31:23 -0500 Subject: [PATCH 20/24] Fixes uses of process_input --- nova/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 7ddf056ea..0937522ec 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -131,7 +131,7 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): process_input=kwargs.get('process_input', None) addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', True) + check_exit_code=kwargs.get('check_exit_code', 0) stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) @@ -151,7 +151,7 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: + if check_exit_code is not None and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, From a9c5f7dad8cc80046650d7bfc9e9673c13a82dc3 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Wed, 9 Mar 2011 15:08:11 -0500 Subject: [PATCH 21/24] renaming wsgi.Request.best_match to best_match_content_type; correcting calls to that function in code from trunk --- nova/wsgi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index c3e08522d..2d18da8fb 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -85,7 +85,7 @@ class Server(object): class Request(webob.Request): - def best_match(self): + def best_match_content_type(self): """ Determine the most acceptable content-type based on the query extension then the Accept header @@ -354,7 +354,7 @@ class Controller(object): result = method(**arg_dict) if type(result) is dict: - content_type = req.best_match() + content_type = req.best_match_content_type() body = self._serialize(result, content_type) response = webob.Response() From f8a7c12219e8a02c88ddfa0b4b83f0bee37d39c1 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 15:33:20 -0500 Subject: [PATCH 22/24] execvp passes pep8 --- nova/tests/test_virt.py | 2 +- nova/utils.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 7f1ad002e..dfa607f14 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -316,7 +316,7 @@ class IptablesFirewallTestCase(test.TestCase): # self.fw.add_instance(instance_ref) def fake_iptables_execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) + process_input = kwargs.get('process_input', None) if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): return '\n'.join(self.in6_rules), None if cmd == ('sudo', 'iptables-save', '-t', 'filter'): diff --git a/nova/utils.py b/nova/utils.py index 0937522ec..3a4ec3c6a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -40,7 +40,7 @@ import netaddr from eventlet import event from eventlet import greenthread from eventlet.green import subprocess - +None from nova import exception from nova.exception import ProcessExecutionError from nova import log as logging @@ -129,13 +129,13 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) - addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', 0) - stdin=kwargs.get('stdin', subprocess.PIPE) - stdout=kwargs.get('stdout', subprocess.PIPE) - stderr=kwargs.get('stderr', subprocess.PIPE) - cmd=map(str,cmd) + process_input = kwargs.get('process_input', None) + addl_env = kwargs.get('addl_env', None) + check_exit_code = kwargs.get('check_exit_code', 0) + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + cmd = map(str, cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() @@ -151,7 +151,8 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code is not None and obj.returncode != check_exit_code: + if type(check_exit_code) == types.IntType \ + and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, From af6cda6a5672192a0907e1f59dafa550bdd1a3ca Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 21:43:35 +0000 Subject: [PATCH 23/24] remove the semaphore when there is no one waiting on it --- nova/tests/test_virt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 113632a0c..56a271365 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -23,7 +23,6 @@ from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db from nova import flags -from nova import log as logging from nova import test from nova import utils from nova.api.ec2 import cloud @@ -70,11 +69,13 @@ class CacheConcurrencyTestCase(test.TestCase): eventlet.sleep(0) try: self.assertFalse(done2.ready()) + self.assertTrue('fname' in conn._image_sems) finally: wait1.send() done1.wait() eventlet.sleep(0) self.assertTrue(done2.ready()) + self.assertFalse('fname' in conn._image_sems) def test_different_fname_concurrency(self): """Ensures that two different fname caches are concurrent""" From 17147f02abdb50f6df6398c8c3c750d858c1c758 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 17:22:54 -0500 Subject: [PATCH 24/24] execvp: fix docs --- doc/ext/nova_autodoc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/ext/nova_autodoc.py b/doc/ext/nova_autodoc.py index 5429bb656..3dd992d84 100644 --- a/doc/ext/nova_autodoc.py +++ b/doc/ext/nova_autodoc.py @@ -8,5 +8,6 @@ from nova import utils def setup(app): rootdir = os.path.abspath(app.srcdir + '/..') print "**Autodocumenting from %s" % rootdir - rv = utils.execute('cd %s && ./generate_autodoc_index.sh' % rootdir) + os.chdir(rootdir) + rv = utils.execute('./generate_autodoc_index.sh') print rv[0]