From d0b9b2721045454bcb076537af97be9b14a1103d Mon Sep 17 00:00:00 2001 From: "Walter A. Boring IV" Date: Wed, 20 Mar 2013 13:18:12 -0700 Subject: [PATCH] Update Cinder's latest copy of OSLO grizzly stable This patch updates all the files from oslo under stable/grizzly but policy.py. policy.py is a high risk change at this date and it has lots of changes since the last time we updated it. Fixes bug #1157126 Change-Id: I399a1cd8474f718ed5196def90d2fea546fb01f5 (cherry picked from commit eae0264782818342c222c18b549e9cda354c867b) --- bin/cinder-rootwrap | 2 +- bin/cinder-rpc-zmq-receiver | 2 +- cinder/openstack/common/context.py | 8 +- cinder/openstack/common/exception.py | 25 ++-- cinder/openstack/common/excutils.py | 2 +- cinder/openstack/common/fileutils.py | 2 +- cinder/openstack/common/importutils.py | 2 +- cinder/openstack/common/jsonutils.py | 37 +++--- cinder/openstack/common/local.py | 13 ++- cinder/openstack/common/lockutils.py | 107 ++++++++++-------- cinder/openstack/common/network_utils.py | 2 +- cinder/openstack/common/notifier/__init__.py | 2 +- cinder/openstack/common/notifier/api.py | 3 +- .../openstack/common/notifier/log_notifier.py | 3 +- .../common/notifier/no_op_notifier.py | 2 +- .../openstack/common/notifier/rpc_notifier.py | 46 ++++++++ .../common/notifier/rpc_notifier2.py | 52 +++++++++ .../common/notifier/test_notifier.py | 2 +- cinder/openstack/common/rootwrap/__init__.py | 2 +- cinder/openstack/common/rootwrap/filters.py | 2 +- cinder/openstack/common/rootwrap/wrapper.py | 2 +- cinder/openstack/common/rpc/amqp.py | 8 +- cinder/openstack/common/rpc/impl_fake.py | 2 +- cinder/openstack/common/rpc/impl_kombu.py | 2 +- cinder/openstack/common/rpc/impl_qpid.py | 2 +- cinder/openstack/common/rpc/impl_zmq.py | 10 ++ cinder/openstack/common/scheduler/filter.py | 2 +- .../common/scheduler/filters/__init__.py | 2 +- .../filters/availability_zone_filter.py | 2 +- .../scheduler/filters/capabilities_filter.py | 2 +- .../scheduler/filters/extra_specs_ops.py | 2 +- .../common/scheduler/filters/json_filter.py | 4 +- cinder/openstack/common/scheduler/weight.py | 2 +- .../common/scheduler/weights/__init__.py | 2 +- cinder/openstack/common/setup.py | 4 +- cinder/openstack/common/strutils.py | 2 +- cinder/openstack/common/timeutils.py | 20 ++-- cinder/openstack/common/version.py | 2 +- etc/cinder/cinder.conf.sample | 2 +- tools/flakes.py | 19 +++- tools/install_venv_common.py | 2 +- 41 files changed, 274 insertions(+), 137 deletions(-) create mode 100644 cinder/openstack/common/notifier/rpc_notifier.py create mode 100644 cinder/openstack/common/notifier/rpc_notifier2.py diff --git a/bin/cinder-rootwrap b/bin/cinder-rootwrap index 959c9ff9095..cb2a92f5eaf 100755 --- a/bin/cinder-rootwrap +++ b/bin/cinder-rootwrap @@ -1,7 +1,7 @@ #!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/bin/cinder-rpc-zmq-receiver b/bin/cinder-rpc-zmq-receiver index 0e73727d97a..e8dbf0effe0 100755 --- a/bin/cinder-rpc-zmq-receiver +++ b/bin/cinder-rpc-zmq-receiver @@ -1,7 +1,7 @@ #!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC +# Copyright 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain diff --git a/cinder/openstack/common/context.py b/cinder/openstack/common/context.py index dd7dd04c38a..e9cfd73cc11 100644 --- a/cinder/openstack/common/context.py +++ b/cinder/openstack/common/context.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -37,9 +37,9 @@ class RequestContext(object): accesses the system, as well as additional request information. """ - def __init__(self, auth_tok=None, user=None, tenant=None, is_admin=False, + def __init__(self, auth_token=None, user=None, tenant=None, is_admin=False, read_only=False, show_deleted=False, request_id=None): - self.auth_tok = auth_tok + self.auth_token = auth_token self.user = user self.tenant = tenant self.is_admin = is_admin @@ -55,7 +55,7 @@ class RequestContext(object): 'is_admin': self.is_admin, 'read_only': self.read_only, 'show_deleted': self.show_deleted, - 'auth_token': self.auth_tok, + 'auth_token': self.auth_token, 'request_id': self.request_id} diff --git a/cinder/openstack/common/exception.py b/cinder/openstack/common/exception.py index ba32da550b0..c8690157f57 100644 --- a/cinder/openstack/common/exception.py +++ b/cinder/openstack/common/exception.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -21,17 +21,9 @@ Exceptions common to OpenStack projects import logging +from cinder.openstack.common.gettextutils import _ -class ProcessExecutionError(IOError): - def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None): - if description is None: - description = "Unexpected error while running command." - if exit_code is None: - exit_code = '-' - message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( - description, cmd, exit_code, stdout, stderr) - IOError.__init__(self, message) +_FATAL_EXCEPTION_FORMAT_ERRORS = False class Error(Exception): @@ -109,7 +101,7 @@ def wrap_exception(f): except Exception, e: if not isinstance(e, Error): #exc_type, exc_value, exc_traceback = sys.exc_info() - logging.exception('Uncaught exception') + logging.exception(_('Uncaught exception')) #logging.error(traceback.extract_stack(exc_traceback)) raise Error(str(e)) raise @@ -131,9 +123,12 @@ class OpenstackException(Exception): try: self._error_string = self.message % kwargs - except Exception: - # at least get the core message out if something happened - self._error_string = self.message + except Exception as e: + if _FATAL_EXCEPTION_FORMAT_ERRORS: + raise e + else: + # at least get the core message out if something happened + self._error_string = self.message def __str__(self): return self._error_string diff --git a/cinder/openstack/common/excutils.py b/cinder/openstack/common/excutils.py index 9cfabcf4a85..ccb2d072e9c 100644 --- a/cinder/openstack/common/excutils.py +++ b/cinder/openstack/common/excutils.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # Copyright 2012, Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/fileutils.py b/cinder/openstack/common/fileutils.py index 4746ad49814..b988ad03d5c 100644 --- a/cinder/openstack/common/fileutils.py +++ b/cinder/openstack/common/fileutils.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/importutils.py b/cinder/openstack/common/importutils.py index 9dec764fb40..3bd277f47e2 100644 --- a/cinder/openstack/common/importutils.py +++ b/cinder/openstack/common/importutils.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/jsonutils.py b/cinder/openstack/common/jsonutils.py index d3026744407..1e9262ed0df 100644 --- a/cinder/openstack/common/jsonutils.py +++ b/cinder/openstack/common/jsonutils.py @@ -34,6 +34,7 @@ This module provides a few things: import datetime +import functools import inspect import itertools import json @@ -42,7 +43,8 @@ import xmlrpclib from cinder.openstack.common import timeutils -def to_primitive(value, convert_instances=False, level=0): +def to_primitive(value, convert_instances=False, convert_datetime=True, + level=0, max_depth=3): """Convert a complex object into primitives. Handy for JSON serialization. We can optionally handle instances, @@ -78,12 +80,17 @@ def to_primitive(value, convert_instances=False, level=0): if getattr(value, '__module__', None) == 'mox': return 'mock' - if level > 3: + if level > max_depth: return '?' # The try block may not be necessary after the class check above, # but just in case ... try: + recursive = functools.partial(to_primitive, + convert_instances=convert_instances, + convert_datetime=convert_datetime, + level=level, + max_depth=max_depth) # It's not clear why xmlrpclib created their own DateTime type, but # for our purposes, make it a datetime type which is explicitly # handled @@ -91,33 +98,19 @@ def to_primitive(value, convert_instances=False, level=0): value = datetime.datetime(*tuple(value.timetuple())[:6]) if isinstance(value, (list, tuple)): - o = [] - for v in value: - o.append(to_primitive(v, convert_instances=convert_instances, - level=level)) - return o + return [recursive(v) for v in value] elif isinstance(value, dict): - o = {} - for k, v in value.iteritems(): - o[k] = to_primitive(v, convert_instances=convert_instances, - level=level) - return o - elif isinstance(value, datetime.datetime): + return dict((k, recursive(v)) for k, v in value.iteritems()) + elif convert_datetime and isinstance(value, datetime.datetime): return timeutils.strtime(value) elif hasattr(value, 'iteritems'): - return to_primitive(dict(value.iteritems()), - convert_instances=convert_instances, - level=level + 1) + return recursive(dict(value.iteritems()), level=level + 1) elif hasattr(value, '__iter__'): - return to_primitive(list(value), - convert_instances=convert_instances, - level=level) + return recursive(list(value)) elif convert_instances and hasattr(value, '__dict__'): # Likely an instance of something. Watch for cycles. # Ignore class member vars. - return to_primitive(value.__dict__, - convert_instances=convert_instances, - level=level + 1) + return recursive(value.__dict__, level=level + 1) else: return value except TypeError: diff --git a/cinder/openstack/common/local.py b/cinder/openstack/common/local.py index 19d962732c1..f1bfc824bf6 100644 --- a/cinder/openstack/common/local.py +++ b/cinder/openstack/common/local.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -26,6 +26,9 @@ class WeakLocal(corolocal.local): def __getattribute__(self, attr): rval = corolocal.local.__getattribute__(self, attr) if rval: + # NOTE(mikal): this bit is confusing. What is stored is a weak + # reference, not the value itself. We therefore need to lookup + # the weak reference and return the inner value here. rval = rval() return rval @@ -34,4 +37,12 @@ class WeakLocal(corolocal.local): return corolocal.local.__setattr__(self, attr, value) +# NOTE(mikal): the name "store" should be deprecated in the future store = WeakLocal() + +# A "weak" store uses weak references and allows an object to fall out of scope +# when it falls out of scope in the code that uses the thread local storage. A +# "strong" store will hold a reference to the object so that it never falls out +# of scope. +weak_store = WeakLocal() +strong_store = corolocal.local diff --git a/cinder/openstack/common/lockutils.py b/cinder/openstack/common/lockutils.py index 7b0b22a0194..f5d260c213e 100644 --- a/cinder/openstack/common/lockutils.py +++ b/cinder/openstack/common/lockutils.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -29,8 +29,10 @@ from oslo.config import cfg from cinder.openstack.common import fileutils from cinder.openstack.common.gettextutils import _ +from cinder.openstack.common import local from cinder.openstack.common import log as logging + LOG = logging.getLogger(__name__) @@ -38,9 +40,8 @@ util_opts = [ cfg.BoolOpt('disable_process_locking', default=False, help='Whether to disable inter-process locks'), cfg.StrOpt('lock_path', - default=os.path.abspath(os.path.join(os.path.dirname(__file__), - '../')), - help='Directory to use for lock files') + help=('Directory to use for lock files. Default to a ' + 'temp directory')) ] @@ -106,10 +107,10 @@ class _InterProcessLock(object): class _WindowsLock(_InterProcessLock): def trylock(self): - msvcrt.locking(self.lockfile, msvcrt.LK_NBLCK, 1) + msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) def unlock(self): - msvcrt.locking(self.lockfile, msvcrt.LK_UNLCK, 1) + msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) class _PosixLock(_InterProcessLock): @@ -139,7 +140,7 @@ def synchronized(name, lock_file_prefix, external=False, lock_path=None): def foo(self, *args): ... - ensures that only one thread will execute the bar method at a time. + ensures that only one thread will execute the foo method at a time. Different methods can share the same lock:: @@ -183,54 +184,66 @@ def synchronized(name, lock_file_prefix, external=False, lock_path=None): LOG.debug(_('Got semaphore "%(lock)s" for method ' '"%(method)s"...'), {'lock': name, 'method': f.__name__}) - if external and not CONF.disable_process_locking: - LOG.debug(_('Attempting to grab file lock "%(lock)s" for ' - 'method "%(method)s"...'), - {'lock': name, 'method': f.__name__}) - cleanup_dir = False - # We need a copy of lock_path because it is non-local - local_lock_path = lock_path - if not local_lock_path: - local_lock_path = CONF.lock_path + # NOTE(mikal): I know this looks odd + if not hasattr(local.strong_store, 'locks_held'): + local.strong_store.locks_held = [] + local.strong_store.locks_held.append(name) - if not local_lock_path: - cleanup_dir = True - local_lock_path = tempfile.mkdtemp() + try: + if external and not CONF.disable_process_locking: + LOG.debug(_('Attempting to grab file lock "%(lock)s" ' + 'for method "%(method)s"...'), + {'lock': name, 'method': f.__name__}) + cleanup_dir = False - if not os.path.exists(local_lock_path): - cleanup_dir = True - fileutils.ensure_tree(local_lock_path) + # We need a copy of lock_path because it is non-local + local_lock_path = lock_path + if not local_lock_path: + local_lock_path = CONF.lock_path - # NOTE(mikal): the lock name cannot contain directory - # separators - safe_name = name.replace(os.sep, '_') - lock_file_name = '%s%s' % (lock_file_prefix, safe_name) - lock_file_path = os.path.join(local_lock_path, - lock_file_name) + if not local_lock_path: + cleanup_dir = True + local_lock_path = tempfile.mkdtemp() - try: - lock = InterProcessLock(lock_file_path) - with lock: - LOG.debug(_('Got file lock "%(lock)s" at %(path)s ' - 'for method "%(method)s"...'), + if not os.path.exists(local_lock_path): + cleanup_dir = True + fileutils.ensure_tree(local_lock_path) + + # NOTE(mikal): the lock name cannot contain directory + # separators + safe_name = name.replace(os.sep, '_') + lock_file_name = '%s%s' % (lock_file_prefix, safe_name) + lock_file_path = os.path.join(local_lock_path, + lock_file_name) + + try: + lock = InterProcessLock(lock_file_path) + with lock: + LOG.debug(_('Got file lock "%(lock)s" at ' + '%(path)s for method ' + '"%(method)s"...'), + {'lock': name, + 'path': lock_file_path, + 'method': f.__name__}) + retval = f(*args, **kwargs) + finally: + LOG.debug(_('Released file lock "%(lock)s" at ' + '%(path)s for method "%(method)s"...'), {'lock': name, 'path': lock_file_path, 'method': f.__name__}) - retval = f(*args, **kwargs) - finally: - LOG.debug(_('Released file lock "%(lock)s" at %(path)s' - ' for method "%(method)s"...'), - {'lock': name, - 'path': lock_file_path, - 'method': f.__name__}) - # NOTE(vish): This removes the tempdir if we needed - # to create one. This is used to cleanup - # the locks left behind by unit tests. - if cleanup_dir: - shutil.rmtree(local_lock_path) - else: - retval = f(*args, **kwargs) + # NOTE(vish): This removes the tempdir if we needed + # to create one. This is used to + # cleanup the locks left behind by unit + # tests. + if cleanup_dir: + shutil.rmtree(local_lock_path) + else: + retval = f(*args, **kwargs) + + finally: + local.strong_store.locks_held.remove(name) return retval return inner diff --git a/cinder/openstack/common/network_utils.py b/cinder/openstack/common/network_utils.py index 69f67321636..5224e01aa94 100644 --- a/cinder/openstack/common/network_utils.py +++ b/cinder/openstack/common/network_utils.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2012 OpenStack LLC. +# Copyright 2012 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/notifier/__init__.py b/cinder/openstack/common/notifier/__init__.py index 482d54e4fdd..45c3b46ae93 100644 --- a/cinder/openstack/common/notifier/__init__.py +++ b/cinder/openstack/common/notifier/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/notifier/api.py b/cinder/openstack/common/notifier/api.py index aa6c87d0294..38b2daba738 100644 --- a/cinder/openstack/common/notifier/api.py +++ b/cinder/openstack/common/notifier/api.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -24,6 +24,7 @@ from cinder.openstack.common import jsonutils from cinder.openstack.common import log as logging from cinder.openstack.common import timeutils + LOG = logging.getLogger(__name__) notifier_opts = [ diff --git a/cinder/openstack/common/notifier/log_notifier.py b/cinder/openstack/common/notifier/log_notifier.py index 800ab5dccf9..010d29cea3d 100644 --- a/cinder/openstack/common/notifier/log_notifier.py +++ b/cinder/openstack/common/notifier/log_notifier.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -18,6 +18,7 @@ from oslo.config import cfg from cinder.openstack.common import jsonutils from cinder.openstack.common import log as logging + CONF = cfg.CONF diff --git a/cinder/openstack/common/notifier/no_op_notifier.py b/cinder/openstack/common/notifier/no_op_notifier.py index ee1ddbdcac3..bc7a56ca7ac 100644 --- a/cinder/openstack/common/notifier/no_op_notifier.py +++ b/cinder/openstack/common/notifier/no_op_notifier.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/notifier/rpc_notifier.py b/cinder/openstack/common/notifier/rpc_notifier.py new file mode 100644 index 00000000000..46a95a17c95 --- /dev/null +++ b/cinder/openstack/common/notifier/rpc_notifier.py @@ -0,0 +1,46 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +from cinder.openstack.common import context as req_context +from cinder.openstack.common.gettextutils import _ +from cinder.openstack.common import log as logging +from cinder.openstack.common import rpc + +LOG = logging.getLogger(__name__) + +notification_topic_opt = cfg.ListOpt( + 'notification_topics', default=['notifications', ], + help='AMQP topic used for openstack notifications') + +CONF = cfg.CONF +CONF.register_opt(notification_topic_opt) + + +def notify(context, message): + """Sends a notification via RPC""" + if not context: + context = req_context.get_admin_context() + priority = message.get('priority', + CONF.default_notification_level) + priority = priority.lower() + for topic in CONF.notification_topics: + topic = '%s.%s' % (topic, priority) + try: + rpc.notify(context, topic, message) + except Exception: + LOG.exception(_("Could not send notification to %(topic)s. " + "Payload=%(message)s"), locals()) diff --git a/cinder/openstack/common/notifier/rpc_notifier2.py b/cinder/openstack/common/notifier/rpc_notifier2.py new file mode 100644 index 00000000000..62a8eda53db --- /dev/null +++ b/cinder/openstack/common/notifier/rpc_notifier2.py @@ -0,0 +1,52 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +'''messaging based notification driver, with message envelopes''' + +from oslo.config import cfg + +from cinder.openstack.common import context as req_context +from cinder.openstack.common.gettextutils import _ +from cinder.openstack.common import log as logging +from cinder.openstack.common import rpc + +LOG = logging.getLogger(__name__) + +notification_topic_opt = cfg.ListOpt( + 'topics', default=['notifications', ], + help='AMQP topic(s) used for openstack notifications') + +opt_group = cfg.OptGroup(name='rpc_notifier2', + title='Options for rpc_notifier2') + +CONF = cfg.CONF +CONF.register_group(opt_group) +CONF.register_opt(notification_topic_opt, opt_group) + + +def notify(context, message): + """Sends a notification via RPC""" + if not context: + context = req_context.get_admin_context() + priority = message.get('priority', + CONF.default_notification_level) + priority = priority.lower() + for topic in CONF.rpc_notifier2.topics: + topic = '%s.%s' % (topic, priority) + try: + rpc.notify(context, topic, message, envelope=True) + except Exception: + LOG.exception(_("Could not send notification to %(topic)s. " + "Payload=%(message)s"), locals()) diff --git a/cinder/openstack/common/notifier/test_notifier.py b/cinder/openstack/common/notifier/test_notifier.py index 5e348803dc1..96c1746bf42 100644 --- a/cinder/openstack/common/notifier/test_notifier.py +++ b/cinder/openstack/common/notifier/test_notifier.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/rootwrap/__init__.py b/cinder/openstack/common/rootwrap/__init__.py index 671d3c173e2..2d32e4ef318 100644 --- a/cinder/openstack/common/rootwrap/__init__.py +++ b/cinder/openstack/common/rootwrap/__init__.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/rootwrap/filters.py b/cinder/openstack/common/rootwrap/filters.py index 905bbabea76..eadda256ca8 100644 --- a/cinder/openstack/common/rootwrap/filters.py +++ b/cinder/openstack/common/rootwrap/filters.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/rootwrap/wrapper.py b/cinder/openstack/common/rootwrap/wrapper.py index 06b121e659a..a8ab1239059 100644 --- a/cinder/openstack/common/rootwrap/wrapper.py +++ b/cinder/openstack/common/rootwrap/wrapper.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/rpc/amqp.py b/cinder/openstack/common/rpc/amqp.py index de90b528a4d..832511cd73f 100644 --- a/cinder/openstack/common/rpc/amqp.py +++ b/cinder/openstack/common/rpc/amqp.py @@ -443,9 +443,11 @@ class ProxyCallback(_ThreadPoolWithWait): connection_pool=self.connection_pool, log_failure=False) except Exception: - LOG.exception(_('Exception during message handling')) - ctxt.reply(None, sys.exc_info(), - connection_pool=self.connection_pool) + # sys.exc_info() is deleted by LOG.exception(). + exc_info = sys.exc_info() + LOG.error(_('Exception during message handling'), + exc_info=exc_info) + ctxt.reply(None, exc_info, connection_pool=self.connection_pool) class MulticallProxyWaiter(object): diff --git a/cinder/openstack/common/rpc/impl_fake.py b/cinder/openstack/common/rpc/impl_fake.py index dfdcd5d9433..0f2f5350018 100644 --- a/cinder/openstack/common/rpc/impl_fake.py +++ b/cinder/openstack/common/rpc/impl_fake.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC +# Copyright 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain diff --git a/cinder/openstack/common/rpc/impl_kombu.py b/cinder/openstack/common/rpc/impl_kombu.py index 3094a5e7f5f..681f531843d 100644 --- a/cinder/openstack/common/rpc/impl_kombu.py +++ b/cinder/openstack/common/rpc/impl_kombu.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC +# Copyright 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain diff --git a/cinder/openstack/common/rpc/impl_qpid.py b/cinder/openstack/common/rpc/impl_qpid.py index 6a4b4f3ac22..5f181cdd2c5 100644 --- a/cinder/openstack/common/rpc/impl_qpid.py +++ b/cinder/openstack/common/rpc/impl_qpid.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC +# Copyright 2011 OpenStack Foundation # Copyright 2011 - 2012, Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/rpc/impl_zmq.py b/cinder/openstack/common/rpc/impl_zmq.py index 864f873040c..6e3d93bada4 100644 --- a/cinder/openstack/common/rpc/impl_zmq.py +++ b/cinder/openstack/common/rpc/impl_zmq.py @@ -16,6 +16,7 @@ import os import pprint +import re import socket import sys import types @@ -431,6 +432,8 @@ class ZmqProxy(ZmqBaseReactor): def __init__(self, conf): super(ZmqProxy, self).__init__(conf) + pathsep = set((os.path.sep or '', os.path.altsep or '', '/', '\\')) + self.badchars = re.compile(r'[%s]' % re.escape(''.join(pathsep))) self.topic_proxy = {} @@ -456,6 +459,13 @@ class ZmqProxy(ZmqBaseReactor): LOG.info(_("Creating proxy for topic: %s"), topic) try: + # The topic is received over the network, + # don't trust this input. + if self.badchars.search(topic) is not None: + emsg = _("Topic contained dangerous characters.") + LOG.warn(emsg) + raise RPCException(emsg) + out_sock = ZmqSocket("ipc://%s/zmq_topic_%s" % (ipc_dir, topic), sock_type, bind=True) diff --git a/cinder/openstack/common/scheduler/filter.py b/cinder/openstack/common/scheduler/filter.py index 0bdb10d0f6e..52c18afa372 100644 --- a/cinder/openstack/common/scheduler/filter.py +++ b/cinder/openstack/common/scheduler/filter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011-2012 OpenStack, LLC. +# Copyright (c) 2011-2012 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/scheduler/filters/__init__.py b/cinder/openstack/common/scheduler/filters/__init__.py index e47f835ba9b..40bf096a419 100644 --- a/cinder/openstack/common/scheduler/filters/__init__.py +++ b/cinder/openstack/common/scheduler/filters/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/scheduler/filters/availability_zone_filter.py b/cinder/openstack/common/scheduler/filters/availability_zone_filter.py index 0be4bd18ac6..0c3ca1ef74e 100644 --- a/cinder/openstack/common/scheduler/filters/availability_zone_filter.py +++ b/cinder/openstack/common/scheduler/filters/availability_zone_filter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011-2012 OpenStack, LLC. +# Copyright (c) 2011-2012 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/scheduler/filters/capabilities_filter.py b/cinder/openstack/common/scheduler/filters/capabilities_filter.py index cd84460d0d7..41b638a2cb5 100644 --- a/cinder/openstack/common/scheduler/filters/capabilities_filter.py +++ b/cinder/openstack/common/scheduler/filters/capabilities_filter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/scheduler/filters/extra_specs_ops.py b/cinder/openstack/common/scheduler/filters/extra_specs_ops.py index 1565ff292c8..b3730f5af47 100644 --- a/cinder/openstack/common/scheduler/filters/extra_specs_ops.py +++ b/cinder/openstack/common/scheduler/filters/extra_specs_ops.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/scheduler/filters/json_filter.py b/cinder/openstack/common/scheduler/filters/json_filter.py index 0e3bb474ac9..370f23b2ae6 100644 --- a/cinder/openstack/common/scheduler/filters/json_filter.py +++ b/cinder/openstack/common/scheduler/filters/json_filter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -32,7 +32,7 @@ class JsonFilter(filters.BaseHostFilter): if len(args) < 2: return False if op is operator.contains: - bad = not args[0] in args[1:] + bad = args[0] not in args[1:] else: bad = [arg for arg in args[1:] if not op(args[0], arg)] diff --git a/cinder/openstack/common/scheduler/weight.py b/cinder/openstack/common/scheduler/weight.py index c5df9da029e..82f1d25ee3e 100644 --- a/cinder/openstack/common/scheduler/weight.py +++ b/cinder/openstack/common/scheduler/weight.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011-2012 OpenStack, LLC. +# Copyright (c) 2011-2012 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/scheduler/weights/__init__.py b/cinder/openstack/common/scheduler/weights/__init__.py index dd4b33209f9..a2743577d7a 100644 --- a/cinder/openstack/common/scheduler/weights/__init__.py +++ b/cinder/openstack/common/scheduler/weights/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/setup.py b/cinder/openstack/common/setup.py index 80a0ecee88d..030df61c994 100644 --- a/cinder/openstack/common/setup.py +++ b/cinder/openstack/common/setup.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # Copyright 2012-2013 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # @@ -149,7 +149,7 @@ def write_git_changelog(): git_dir = _get_git_directory() if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'): if git_dir: - git_log_cmd = 'git --git-dir=%s log --stat' % git_dir + git_log_cmd = 'git --git-dir=%s log' % git_dir changelog = _run_shell_command(git_log_cmd) mailmap = _parse_git_mailmap(git_dir) with open(new_changelog, "w") as changelog_file: diff --git a/cinder/openstack/common/strutils.py b/cinder/openstack/common/strutils.py index 7813b64225f..ecf3cfdc4da 100644 --- a/cinder/openstack/common/strutils.py +++ b/cinder/openstack/common/strutils.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/cinder/openstack/common/timeutils.py b/cinder/openstack/common/timeutils.py index 5a011e8181f..60943659076 100644 --- a/cinder/openstack/common/timeutils.py +++ b/cinder/openstack/common/timeutils.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -25,18 +25,22 @@ import datetime import iso8601 -TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" -PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" +# ISO 8601 extended time format with microseconds +_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' +_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND -def isotime(at=None): +def isotime(at=None, subsecond=False): """Stringify time in ISO 8601 format""" if not at: at = utcnow() - str = at.strftime(TIME_FORMAT) + st = at.strftime(_ISO8601_TIME_FORMAT + if not subsecond + else _ISO8601_TIME_FORMAT_SUBSECOND) tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' - str += ('Z' if tz == 'UTC' else tz) - return str + st += ('Z' if tz == 'UTC' else tz) + return st def parse_isotime(timestr): @@ -179,4 +183,4 @@ def is_soon(dt, window): :return: True if expiration is within the given duration """ soon = (utcnow() + datetime.timedelta(seconds=window)) - return normalize_time(dt) < soon + return normalize_time(dt) <= soon diff --git a/cinder/openstack/common/version.py b/cinder/openstack/common/version.py index 3ed28e44c0c..cfcd01b5827 100644 --- a/cinder/openstack/common/version.py +++ b/cinder/openstack/common/version.py @@ -1,5 +1,5 @@ -# Copyright 2012 OpenStack LLC +# Copyright 2012 OpenStack Foundation # Copyright 2012-2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/etc/cinder/cinder.conf.sample b/etc/cinder/cinder.conf.sample index d0bd2f4da11..2036f7025ab 100644 --- a/etc/cinder/cinder.conf.sample +++ b/etc/cinder/cinder.conf.sample @@ -381,7 +381,7 @@ #disable_process_locking=false # Directory to use for lock files (string value) -#lock_path=/usr/lib/python/site-packages/cinder/openstack +#lock_path= # diff --git a/tools/flakes.py b/tools/flakes.py index 7f96116ca54..191bd6eabcc 100644 --- a/tools/flakes.py +++ b/tools/flakes.py @@ -4,12 +4,21 @@ Synced in from openstack-common """ + +__all__ = ['main'] + +import __builtin__ as builtins import sys -import pyflakes.checker -from pyflakes.scripts import pyflakes +import pyflakes.api +from pyflakes import checker + + +def main(): + checker.Checker.builtIns = (set(dir(builtins)) | + set(['_']) | + set(checker._MAGIC_GLOBALS)) + sys.exit(pyflakes.api.main()) if __name__ == "__main__": - orig_builtins = set(pyflakes.checker._MAGIC_GLOBALS) - pyflakes.checker._MAGIC_GLOBALS = orig_builtins | set(['_']) - sys.exit(pyflakes.main()) + main() diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py index fca25122990..413065640f4 100644 --- a/tools/install_venv_common.py +++ b/tools/install_venv_common.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2013 OpenStack, LLC +# Copyright 2013 OpenStack Foundation # Copyright 2013 IBM Corp. # # Licensed under the Apache License, Version 2.0 (the "License"); you may