From c9230f2de05eda7ee39256444dcc95db33fff45b Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Wed, 12 Jun 2013 19:37:16 +0200 Subject: [PATCH] Merge from oslo-incubator Change-Id: Ie5d9d2c1938e58125b58e64a3363f1e1e8ce2ba2 --- cinder/openstack/common/context.py | 8 +- cinder/openstack/common/eventlet_backdoor.py | 12 +-- cinder/openstack/common/exception.py | 5 +- cinder/openstack/common/importutils.py | 7 +- cinder/openstack/common/lockutils.py | 19 ++-- cinder/openstack/common/log.py | 7 +- cinder/openstack/common/loopingcall.py | 4 +- cinder/openstack/common/network_utils.py | 7 +- cinder/openstack/common/notifier/api.py | 2 +- .../openstack/common/notifier/log_notifier.py | 4 +- .../common/notifier/no_op_notifier.py | 2 +- .../openstack/common/notifier/rpc_notifier.py | 2 +- .../common/notifier/rpc_notifier2.py | 2 +- cinder/openstack/common/processutils.py | 73 ++++++++++++++- cinder/openstack/common/rootwrap/cmd.py | 4 +- cinder/openstack/common/rootwrap/filters.py | 24 ++--- cinder/openstack/common/rootwrap/wrapper.py | 20 ++--- cinder/openstack/common/service.py | 5 +- cinder/openstack/common/strutils.py | 88 ++++++++++++++++--- cinder/openstack/common/threadgroup.py | 13 ++- cinder/openstack/common/timeutils.py | 23 ++--- tools/install_venv_common.py | 50 ++++++----- tools/patch_tox_venv.py | 16 +++- 23 files changed, 280 insertions(+), 117 deletions(-) diff --git a/cinder/openstack/common/context.py b/cinder/openstack/common/context.py index e9cfd73cc11..7c5f6a76a66 100644 --- a/cinder/openstack/common/context.py +++ b/cinder/openstack/common/context.py @@ -23,16 +23,18 @@ context or provide additional information in their specific WSGI pipeline. """ import itertools -import uuid + +from cinder.openstack.common import uuidutils def generate_request_id(): - return 'req-' + str(uuid.uuid4()) + return 'req-%s' % uuidutils.generate_uuid() class RequestContext(object): - """ + """Helper class to represent useful information about a request context. + Stores information about the security context under which the user accesses the system, as well as additional request information. """ diff --git a/cinder/openstack/common/eventlet_backdoor.py b/cinder/openstack/common/eventlet_backdoor.py index c0ad460fe6e..57b89ae914e 100644 --- a/cinder/openstack/common/eventlet_backdoor.py +++ b/cinder/openstack/common/eventlet_backdoor.py @@ -16,6 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import print_function + import gc import pprint import sys @@ -37,7 +39,7 @@ CONF.register_opts(eventlet_backdoor_opts) def _dont_use_this(): - print "Don't use this, just disconnect instead" + print("Don't use this, just disconnect instead") def _find_objects(t): @@ -46,16 +48,16 @@ def _find_objects(t): def _print_greenthreads(): for i, gt in enumerate(_find_objects(greenlet.greenlet)): - print i, gt + print(i, gt) traceback.print_stack(gt.gr_frame) - print + print() def _print_nativethreads(): for threadId, stack in sys._current_frames().items(): - print threadId + print(threadId) traceback.print_stack(stack) - print + print() def initialize_if_enabled(): diff --git a/cinder/openstack/common/exception.py b/cinder/openstack/common/exception.py index c8690157f57..ecaf275335e 100644 --- a/cinder/openstack/common/exception.py +++ b/cinder/openstack/common/exception.py @@ -98,7 +98,7 @@ def wrap_exception(f): def _wrap(*args, **kw): try: return f(*args, **kw) - except Exception, e: + except Exception as e: if not isinstance(e, Error): #exc_type, exc_value, exc_traceback = sys.exc_info() logging.exception(_('Uncaught exception')) @@ -110,8 +110,7 @@ def wrap_exception(f): class OpenstackException(Exception): - """ - Base Exception + """Base Exception class. To correctly use this class, inherit from it and define a 'message' property. That message will get printf'd diff --git a/cinder/openstack/common/importutils.py b/cinder/openstack/common/importutils.py index 3bd277f47e2..7a303f93f21 100644 --- a/cinder/openstack/common/importutils.py +++ b/cinder/openstack/common/importutils.py @@ -24,7 +24,7 @@ import traceback def import_class(import_str): - """Returns a class from a string including module and class""" + """Returns a class from a string including module and class.""" mod_str, _sep, class_str = import_str.rpartition('.') try: __import__(mod_str) @@ -41,8 +41,9 @@ def import_object(import_str, *args, **kwargs): def import_object_ns(name_space, import_str, *args, **kwargs): - """ - Import a class and return an instance of it, first by trying + """Tries to import object from default namespace. + + Imports a class and return an instance of it, first by trying to find the class in a default namespace, then failing back to a full path if not found in the default namespace. """ diff --git a/cinder/openstack/common/lockutils.py b/cinder/openstack/common/lockutils.py index f21f0d91869..005ad3cf9de 100644 --- a/cinder/openstack/common/lockutils.py +++ b/cinder/openstack/common/lockutils.py @@ -158,17 +158,18 @@ def synchronized(name, lock_file_prefix, external=False, lock_path=None): This way only one of either foo or bar can be executing at a time. - The lock_file_prefix argument is used to provide lock files on disk with a - meaningful prefix. The prefix should end with a hyphen ('-') if specified. + :param lock_file_prefix: The lock_file_prefix argument is used to provide + lock files on disk with a meaningful prefix. The prefix should end with a + hyphen ('-') if specified. - The external keyword argument denotes whether this lock should work across - multiple processes. This means that if two different workers both run a - a method decorated with @synchronized('mylock', external=True), only one - of them will execute at a time. + :param external: The external keyword argument denotes whether this lock + should work across multiple processes. This means that if two different + workers both run a a method decorated with @synchronized('mylock', + external=True), only one of them will execute at a time. - The lock_path keyword argument is used to specify a special location for - external lock files to live. If nothing is set, then CONF.lock_path is - used as a default. + :param lock_path: The lock_path keyword argument is used to specify a + special location for external lock files to live. If nothing is set, then + CONF.lock_path is used as a default. """ def wrap(f): diff --git a/cinder/openstack/common/log.py b/cinder/openstack/common/log.py index 8ba177ab9c6..caea2b87909 100644 --- a/cinder/openstack/common/log.py +++ b/cinder/openstack/common/log.py @@ -459,10 +459,11 @@ def getLogger(name='unknown', version='unknown'): def getLazyLogger(name='unknown', version='unknown'): - """ - create a pass-through logger that does not create the real logger + """Returns lazy logger. + + Creates a pass-through logger that does not create the real logger until it is really needed and delegates all calls to the real logger - once it is created + once it is created. """ return LazyAdapter(name, version) diff --git a/cinder/openstack/common/loopingcall.py b/cinder/openstack/common/loopingcall.py index 8be3a00eb40..f62792d5c75 100644 --- a/cinder/openstack/common/loopingcall.py +++ b/cinder/openstack/common/loopingcall.py @@ -84,7 +84,7 @@ class FixedIntervalLoopingCall(LoopingCallBase): LOG.warn(_('task run outlasted interval by %s sec') % -delay) greenthread.sleep(delay if delay > 0 else 0) - except LoopingCallDone, e: + except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: @@ -131,7 +131,7 @@ class DynamicLoopingCall(LoopingCallBase): LOG.debug(_('Dynamic looping call sleeping for %.02f ' 'seconds'), idle) greenthread.sleep(idle) - except LoopingCallDone, e: + except LoopingCallDone as e: self.stop() done.send(e.retvalue) except Exception: diff --git a/cinder/openstack/common/network_utils.py b/cinder/openstack/common/network_utils.py index 5224e01aa94..c94cae6e034 100644 --- a/cinder/openstack/common/network_utils.py +++ b/cinder/openstack/common/network_utils.py @@ -19,14 +19,15 @@ Network-related utilities and helper functions. """ -import logging +from cinder.openstack.common import log as logging + LOG = logging.getLogger(__name__) def parse_host_port(address, default_port=None): - """ - Interpret a string as a host:port pair. + """Interpret a string as a host:port pair. + An IPv6 address MUST be escaped if accompanied by a port, because otherwise ambiguity ensues: 2001:db8:85a3::8a2e:370:7334 means both [2001:db8:85a3::8a2e:370:7334] and diff --git a/cinder/openstack/common/notifier/api.py b/cinder/openstack/common/notifier/api.py index 6b82e4451e0..f1115a7438f 100644 --- a/cinder/openstack/common/notifier/api.py +++ b/cinder/openstack/common/notifier/api.py @@ -56,7 +56,7 @@ class BadPriorityException(Exception): def notify_decorator(name, fn): - """ decorator for notify which is used from utils.monkey_patch() + """Decorator for notify which is used from utils.monkey_patch(). :param name: name of the function :param function: - object of the function diff --git a/cinder/openstack/common/notifier/log_notifier.py b/cinder/openstack/common/notifier/log_notifier.py index 010d29cea3d..8d81c24c30b 100644 --- a/cinder/openstack/common/notifier/log_notifier.py +++ b/cinder/openstack/common/notifier/log_notifier.py @@ -24,7 +24,9 @@ CONF = cfg.CONF def notify(_context, message): """Notifies the recipient of the desired event given the model. - Log notifications using openstack's default logging system""" + + Log notifications using openstack's default logging system. + """ priority = message.get('priority', CONF.default_notification_level) diff --git a/cinder/openstack/common/notifier/no_op_notifier.py b/cinder/openstack/common/notifier/no_op_notifier.py index bc7a56ca7ac..13d946e362d 100644 --- a/cinder/openstack/common/notifier/no_op_notifier.py +++ b/cinder/openstack/common/notifier/no_op_notifier.py @@ -15,5 +15,5 @@ def notify(_context, message): - """Notifies the recipient of the desired event given the model""" + """Notifies the recipient of the desired event given the model.""" pass diff --git a/cinder/openstack/common/notifier/rpc_notifier.py b/cinder/openstack/common/notifier/rpc_notifier.py index 46a95a17c95..32a8954a208 100644 --- a/cinder/openstack/common/notifier/rpc_notifier.py +++ b/cinder/openstack/common/notifier/rpc_notifier.py @@ -31,7 +31,7 @@ CONF.register_opt(notification_topic_opt) def notify(context, message): - """Sends a notification via RPC""" + """Sends a notification via RPC.""" if not context: context = req_context.get_admin_context() priority = message.get('priority', diff --git a/cinder/openstack/common/notifier/rpc_notifier2.py b/cinder/openstack/common/notifier/rpc_notifier2.py index 62a8eda53db..d624451def3 100644 --- a/cinder/openstack/common/notifier/rpc_notifier2.py +++ b/cinder/openstack/common/notifier/rpc_notifier2.py @@ -37,7 +37,7 @@ CONF.register_opt(notification_topic_opt, opt_group) def notify(context, message): - """Sends a notification via RPC""" + """Sends a notification via RPC.""" if not context: context = req_context.get_admin_context() priority = message.get('priority', diff --git a/cinder/openstack/common/processutils.py b/cinder/openstack/common/processutils.py index 47e7d08b1ed..dd6e73122d1 100644 --- a/cinder/openstack/common/processutils.py +++ b/cinder/openstack/common/processutils.py @@ -34,6 +34,11 @@ from cinder.openstack.common import log as logging LOG = logging.getLogger(__name__) +class InvalidArgumentError(Exception): + def __init__(self, message=None): + super(InvalidArgumentError, self).__init__(message) + + class UnknownArgumentError(Exception): def __init__(self, message=None): super(UnknownArgumentError, self).__init__(message) @@ -69,9 +74,9 @@ def _subprocess_setup(): def execute(*cmd, **kwargs): - """ - Helper method to shell out and execute a command through subprocess with - optional retry. + """Helper method to shell out and execute a command through subprocess. + + Allows optional retry. :param cmd: Passed to subprocess.Popen. :type cmd: string @@ -118,7 +123,7 @@ def execute(*cmd, **kwargs): elif isinstance(check_exit_code, int): check_exit_code = [check_exit_code] - if len(kwargs): + if kwargs: raise UnknownArgumentError(_('Got unknown keyword args ' 'to utils.execute: %r') % kwargs) @@ -179,3 +184,63 @@ def execute(*cmd, **kwargs): # call clean something up in between calls, without # it two execute calls in a row hangs the second one greenthread.sleep(0) + + +def trycmd(*args, **kwargs): + """A wrapper around execute() to more easily handle warnings and errors. + + Returns an (out, err) tuple of strings containing the output of + the command's stdout and stderr. If 'err' is not empty then the + command can be considered to have failed. + + :discard_warnings True | False. Defaults to False. If set to True, + then for succeeding commands, stderr is cleared + + """ + discard_warnings = kwargs.pop('discard_warnings', False) + + try: + out, err = execute(*args, **kwargs) + failed = False + except ProcessExecutionError as exn: + out, err = '', str(exn) + failed = True + + if not failed and discard_warnings and err: + # Handle commands that output to stderr but otherwise succeed + err = '' + + return out, err + + +def ssh_execute(ssh, cmd, process_input=None, + addl_env=None, check_exit_code=True): + LOG.debug(_('Running cmd (SSH): %s'), cmd) + if addl_env: + raise InvalidArgumentError(_('Environment not supported over SSH')) + + if process_input: + # This is (probably) fixable if we need it... + raise InvalidArgumentError(_('process_input not supported over SSH')) + + stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) + channel = stdout_stream.channel + + # NOTE(justinsb): This seems suspicious... + # ...other SSH clients have buffering issues with this approach + stdout = stdout_stream.read() + stderr = stderr_stream.read() + stdin_stream.close() + + exit_status = channel.recv_exit_status() + + # exit_status == -1 if no exit code was returned + if exit_status != -1: + LOG.debug(_('Result was %s') % exit_status) + if check_exit_code and exit_status != 0: + raise ProcessExecutionError(exit_code=exit_status, + stdout=stdout, + stderr=stderr, + cmd=cmd) + + return (stdout, stderr) diff --git a/cinder/openstack/common/rootwrap/cmd.py b/cinder/openstack/common/rootwrap/cmd.py index 78265e30cab..b2c69eb943a 100755 --- a/cinder/openstack/common/rootwrap/cmd.py +++ b/cinder/openstack/common/rootwrap/cmd.py @@ -33,6 +33,8 @@ they are needed, to avoid allowing more than is necessary. """ +from __future__ import print_function + import ConfigParser import logging import os @@ -55,7 +57,7 @@ def _subprocess_setup(): def _exit_error(execname, message, errorcode, log=True): - print "%s: %s" % (execname, message) + print("%s: %s" % (execname, message)) if log: logging.error(message) sys.exit(errorcode) diff --git a/cinder/openstack/common/rootwrap/filters.py b/cinder/openstack/common/rootwrap/filters.py index d9618af8834..ae7c62cada4 100644 --- a/cinder/openstack/common/rootwrap/filters.py +++ b/cinder/openstack/common/rootwrap/filters.py @@ -20,7 +20,7 @@ import re class CommandFilter(object): - """Command filter only checking that the 1st argument matches exec_path""" + """Command filter only checking that the 1st argument matches exec_path.""" def __init__(self, exec_path, run_as, *args): self.name = '' @@ -30,7 +30,7 @@ class CommandFilter(object): self.real_exec = None def get_exec(self, exec_dirs=[]): - """Returns existing executable, or empty string if none found""" + """Returns existing executable, or empty string if none found.""" if self.real_exec is not None: return self.real_exec self.real_exec = "" @@ -46,10 +46,8 @@ class CommandFilter(object): return self.real_exec def match(self, userargs): - """Only check that the first argument (command) matches exec_path""" - if (os.path.basename(self.exec_path) == userargs[0]): - return True - return False + """Only check that the first argument (command) matches exec_path.""" + return os.path.basename(self.exec_path) == userargs[0] def get_command(self, userargs, exec_dirs=[]): """Returns command to execute (with sudo -u if run_as != root).""" @@ -60,12 +58,12 @@ class CommandFilter(object): return [to_exec] + userargs[1:] def get_environment(self, userargs): - """Returns specific environment to set, None if none""" + """Returns specific environment to set, None if none.""" return None class RegExpFilter(CommandFilter): - """Command filter doing regexp matching for every argument""" + """Command filter doing regexp matching for every argument.""" def match(self, userargs): # Early skip if command or number of args don't match @@ -135,7 +133,7 @@ class PathFilter(CommandFilter): class DnsmasqFilter(CommandFilter): - """Specific filter for the dnsmasq call (which includes env)""" + """Specific filter for the dnsmasq call (which includes env).""" CONFIG_FILE_ARG = 'CONFIG_FILE' @@ -160,7 +158,7 @@ class DnsmasqFilter(CommandFilter): class DeprecatedDnsmasqFilter(DnsmasqFilter): - """Variant of dnsmasq filter to support old-style FLAGFILE""" + """Variant of dnsmasq filter to support old-style FLAGFILE.""" CONFIG_FILE_ARG = 'FLAGFILE' @@ -196,6 +194,10 @@ class KillFilter(CommandFilter): return False try: command = os.readlink("/proc/%d/exe" % int(args[1])) + # NOTE(yufang521247): /proc/PID/exe may have '\0' on the + # end, because python doen't stop at '\0' when read the + # target path. + command = command.split('\0')[0] # NOTE(dprince): /proc/PID/exe may have ' (deleted)' on # the end if an executable is updated or deleted if command.endswith(" (deleted)"): @@ -210,7 +212,7 @@ class KillFilter(CommandFilter): class ReadFileFilter(CommandFilter): - """Specific filter for the utils.read_file_as_root call""" + """Specific filter for the utils.read_file_as_root call.""" def __init__(self, file_path, *args): self.file_path = file_path diff --git a/cinder/openstack/common/rootwrap/wrapper.py b/cinder/openstack/common/rootwrap/wrapper.py index a8ab1239059..15af99476f6 100644 --- a/cinder/openstack/common/rootwrap/wrapper.py +++ b/cinder/openstack/common/rootwrap/wrapper.py @@ -31,10 +31,7 @@ class NoFilterMatched(Exception): class FilterMatchNotExecutable(Exception): - """ - This exception is raised when a filter matched but no executable was - found. - """ + """Raised when a filter matched but no executable was found.""" def __init__(self, match=None, **kwargs): self.match = match @@ -93,7 +90,7 @@ def setup_syslog(execname, facility, level): def build_filter(class_name, *args): - """Returns a filter object of class class_name""" + """Returns a filter object of class class_name.""" if not hasattr(filters, class_name): logging.warning("Skipping unknown filter class (%s) specified " "in filter definitions" % class_name) @@ -103,7 +100,7 @@ def build_filter(class_name, *args): def load_filters(filters_path): - """Load filters from a list of directories""" + """Load filters from a list of directories.""" filterlist = [] for filterdir in filters_path: if not os.path.isdir(filterdir): @@ -121,17 +118,18 @@ def load_filters(filters_path): return filterlist -def match_filter(filters, userargs, exec_dirs=[]): - """ - Checks user command and arguments through command filters and - returns the first matching filter. +def match_filter(filter_list, userargs, exec_dirs=[]): + """Checks user command and arguments through command filters. + + Returns the first matching filter. + Raises NoFilterMatched if no filter matched. Raises FilterMatchNotExecutable if no executable was found for the best filter match. """ first_not_executable_filter = None - for f in filters: + for f in filter_list: if f.match(userargs): # Try other filters if executable is absent if not f.get_exec(exec_dirs=exec_dirs): diff --git a/cinder/openstack/common/service.py b/cinder/openstack/common/service.py index 8600a0b087f..7cbd3690a64 100644 --- a/cinder/openstack/common/service.py +++ b/cinder/openstack/common/service.py @@ -52,7 +52,7 @@ class Launcher(object): """ self._services = threadgroup.ThreadGroup() - eventlet_backdoor.initialize_if_enabled() + self.backdoor_port = eventlet_backdoor.initialize_if_enabled() @staticmethod def run_service(service): @@ -72,6 +72,7 @@ class Launcher(object): :returns: None """ + service.backdoor_port = self.backdoor_port self._services.add_thread(self.run_service, service) def stop(self): @@ -270,7 +271,7 @@ class ProcessLauncher(object): return wrap def wait(self): - """Loop waiting on children to die and respawning as necessary""" + """Loop waiting on children to die and respawning as necessary.""" LOG.debug(_('Full set of CONF:')) CONF.log_opt_values(LOG, std_logging.DEBUG) diff --git a/cinder/openstack/common/strutils.py b/cinder/openstack/common/strutils.py index a4e38997254..2f93610dd54 100644 --- a/cinder/openstack/common/strutils.py +++ b/cinder/openstack/common/strutils.py @@ -19,18 +19,33 @@ System-level utilities and helper functions. """ +import re import sys +import unicodedata from cinder.openstack.common.gettextutils import _ +# Used for looking up extensions of text +# to their 'multiplied' byte amount +BYTE_MULTIPLIERS = { + '': 1, + 't': 1024 ** 4, + 'g': 1024 ** 3, + 'm': 1024 ** 2, + 'k': 1024, +} + + TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes') FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no') +SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]") +SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+") + def int_from_bool_as_string(subject): - """ - Interpret a string as a boolean and return either 1 or 0. + """Interpret a string as a boolean and return either 1 or 0. Any string value in: @@ -44,8 +59,7 @@ def int_from_bool_as_string(subject): def bool_from_string(subject, strict=False): - """ - Interpret a string as a boolean. + """Interpret a string as a boolean. A case-insensitive match is performed such that strings matching 't', 'true', 'on', 'y', 'yes', or '1' are considered True and, when @@ -78,9 +92,7 @@ def bool_from_string(subject, strict=False): def safe_decode(text, incoming=None, errors='strict'): - """ - Decodes incoming str using `incoming` if they're - not already unicode. + """Decodes incoming str using `incoming` if they're not already unicode. :param incoming: Text's current encoding :param errors: Errors handling policy. See here for valid @@ -119,11 +131,10 @@ def safe_decode(text, incoming=None, errors='strict'): def safe_encode(text, incoming=None, encoding='utf-8', errors='strict'): - """ - Encodes incoming str/unicode using `encoding`. If - incoming is not specified, text is expected to - be encoded with current python's default encoding. - (`sys.getdefaultencoding`) + """Encodes incoming str/unicode using `encoding`. + + If incoming is not specified, text is expected to be encoded with + current python's default encoding. (`sys.getdefaultencoding`) :param incoming: Text's current encoding :param encoding: Expected encoding for text (Default UTF-8) @@ -148,3 +159,56 @@ def safe_encode(text, incoming=None, return text.encode(encoding, errors) return text + + +def to_bytes(text, default=0): + """Try to turn a string into a number of bytes. Looks at the last + characters of the text to determine what conversion is needed to + turn the input text into a byte number. + + Supports: B/b, K/k, M/m, G/g, T/t (or the same with b/B on the end) + + """ + # Take off everything not number 'like' (which should leave + # only the byte 'identifier' left) + mult_key_org = text.lstrip('-1234567890') + mult_key = mult_key_org.lower() + mult_key_len = len(mult_key) + if mult_key.endswith("b"): + mult_key = mult_key[0:-1] + try: + multiplier = BYTE_MULTIPLIERS[mult_key] + if mult_key_len: + # Empty cases shouldn't cause text[0:-0] + text = text[0:-mult_key_len] + return int(text) * multiplier + except KeyError: + msg = _('Unknown byte multiplier: %s') % mult_key_org + raise TypeError(msg) + except ValueError: + return default + + +def to_slug(value, incoming=None, errors="strict"): + """Normalize string. + + Convert to lowercase, remove non-word characters, and convert spaces + to hyphens. + + Inspired by Django's `slugify` filter. + + :param value: Text to slugify + :param incoming: Text's current encoding + :param errors: Errors handling policy. See here for valid + values http://docs.python.org/2/library/codecs.html + :returns: slugified unicode representation of `value` + :raises TypeError: If text is not an instance of basestring + """ + value = safe_decode(value, incoming, errors) + # NOTE(aababilov): no need to use safe_(encode|decode) here: + # encodings are always "ascii", error handling is always "ignore" + # and types are always known (first: unicode; second: str) + value = unicodedata.normalize("NFKD", value).encode( + "ascii", "ignore").decode("ascii") + value = SLUGIFY_STRIP_RE.sub("", value).strip().lower() + return SLUGIFY_HYPHENATE_RE.sub("-", value) diff --git a/cinder/openstack/common/threadgroup.py b/cinder/openstack/common/threadgroup.py index 5d6ec006b90..7e804160389 100644 --- a/cinder/openstack/common/threadgroup.py +++ b/cinder/openstack/common/threadgroup.py @@ -26,7 +26,7 @@ LOG = logging.getLogger(__name__) def _thread_done(gt, *args, **kwargs): - """ Callback function to be passed to GreenThread.link() when we spawn() + """Callback function to be passed to GreenThread.link() when we spawn() Calls the :class:`ThreadGroup` to notify if. """ @@ -34,7 +34,7 @@ def _thread_done(gt, *args, **kwargs): class Thread(object): - """ Wrapper around a greenthread, that holds a reference to the + """Wrapper around a greenthread, that holds a reference to the :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when it has done so it can be removed from the threads list. """ @@ -50,7 +50,7 @@ class Thread(object): class ThreadGroup(object): - """ The point of the ThreadGroup classis to: + """The point of the ThreadGroup classis to: * keep track of timers and greenthreads (making it easier to stop them when need be). @@ -61,6 +61,13 @@ class ThreadGroup(object): self.threads = [] self.timers = [] + def add_dynamic_timer(self, callback, initial_delay=None, + periodic_interval_max=None, *args, **kwargs): + timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs) + timer.start(initial_delay=initial_delay, + periodic_interval_max=periodic_interval_max) + self.timers.append(timer) + def add_timer(self, interval, callback, initial_delay=None, *args, **kwargs): pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs) diff --git a/cinder/openstack/common/timeutils.py b/cinder/openstack/common/timeutils.py index 60943659076..ac2441bcb41 100644 --- a/cinder/openstack/common/timeutils.py +++ b/cinder/openstack/common/timeutils.py @@ -32,7 +32,7 @@ PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND def isotime(at=None, subsecond=False): - """Stringify time in ISO 8601 format""" + """Stringify time in ISO 8601 format.""" if not at: at = utcnow() st = at.strftime(_ISO8601_TIME_FORMAT @@ -44,7 +44,7 @@ def isotime(at=None, subsecond=False): def parse_isotime(timestr): - """Parse time from ISO 8601 format""" + """Parse time from ISO 8601 format.""" try: return iso8601.parse_date(timestr) except iso8601.ParseError as e: @@ -66,7 +66,7 @@ def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): def normalize_time(timestamp): - """Normalize time in arbitrary timezone to UTC naive object""" + """Normalize time in arbitrary timezone to UTC naive object.""" offset = timestamp.utcoffset() if offset is None: return timestamp @@ -103,7 +103,7 @@ def utcnow(): def iso8601_from_timestamp(timestamp): - """Returns a iso8601 formated date from timestamp""" + """Returns a iso8601 formated date from timestamp.""" return isotime(datetime.datetime.utcfromtimestamp(timestamp)) @@ -111,9 +111,9 @@ utcnow.override_time = None def set_time_override(override_time=datetime.datetime.utcnow()): - """ - Override utils.utcnow to return a constant time or a list thereof, - one at a time. + """Overrides utils.utcnow. + + Make it return a constant time or a list thereof, one at a time. """ utcnow.override_time = override_time @@ -141,7 +141,8 @@ def clear_time_override(): def marshall_now(now=None): """Make an rpc-safe datetime with microseconds. - Note: tzinfo is stripped, but not required for relative times.""" + Note: tzinfo is stripped, but not required for relative times. + """ if not now: now = utcnow() return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, @@ -161,7 +162,8 @@ def unmarshall_time(tyme): def delta_seconds(before, after): - """ + """Return the difference between two timing objects. + Compute the difference in seconds between two date, time, or datetime objects (as a float, to microsecond resolution). """ @@ -174,8 +176,7 @@ def delta_seconds(before, after): def is_soon(dt, window): - """ - Determines if time is going to happen in the next window seconds. + """Determines if time is going to happen in the next window seconds. :params dt: the time :params window: minimum seconds to remain to consider the time not soon diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py index 413065640f4..42a44e8cd26 100644 --- a/tools/install_venv_common.py +++ b/tools/install_venv_common.py @@ -18,10 +18,15 @@ """Provides methods needed by installation script for OpenStack development virtual environments. +Since this script is used to bootstrap a virtualenv from the system's Python +environment, it should be kept strictly compatible with Python 2.6. + Synced in from openstack-common """ -import argparse +from __future__ import print_function + +import optparse import os import subprocess import sys @@ -39,7 +44,7 @@ class InstallVenv(object): self.project = project def die(self, message, *args): - print >> sys.stderr, message % args + print(message % args, file=sys.stderr) sys.exit(1) def check_python_version(self): @@ -86,20 +91,20 @@ class InstallVenv(object): virtual environment. """ if not os.path.isdir(self.venv): - print 'Creating venv...', + print('Creating venv...', end=' ') if no_site_packages: self.run_command(['virtualenv', '-q', '--no-site-packages', self.venv]) else: self.run_command(['virtualenv', '-q', self.venv]) - print 'done.' - print 'Installing pip in venv...', + print('done.') + print('Installing pip in venv...', end=' ') if not self.run_command(['tools/with_venv.sh', 'easy_install', 'pip>1.0']).strip(): self.die("Failed to install pip.") - print 'done.' + print('done.') else: - print "venv already exists..." + print("venv already exists...") pass def pip_install(self, *args): @@ -108,7 +113,7 @@ class InstallVenv(object): redirect_output=False) def install_dependencies(self): - print 'Installing dependencies with pip (this can take a while)...' + print('Installing dependencies with pip (this can take a while)...') # First things first, make sure our venv has the latest pip and # distribute. @@ -131,12 +136,12 @@ class InstallVenv(object): def parse_args(self, argv): """Parses command-line arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument('-n', '--no-site-packages', - action='store_true', - help="Do not inherit packages from global Python " - "install") - return parser.parse_args(argv[1:]) + parser = optparse.OptionParser() + parser.add_option('-n', '--no-site-packages', + action='store_true', + help="Do not inherit packages from global Python " + "install") + return parser.parse_args(argv[1:])[0] class Distro(InstallVenv): @@ -150,12 +155,12 @@ class Distro(InstallVenv): return if self.check_cmd('easy_install'): - print 'Installing virtualenv via easy_install...', + print('Installing virtualenv via easy_install...', end=' ') if self.run_command(['easy_install', 'virtualenv']): - print 'Succeeded' + print('Succeeded') return else: - print 'Failed' + print('Failed') self.die('ERROR: virtualenv not found.\n\n%s development' ' requires virtualenv, please install it using your' @@ -180,10 +185,6 @@ class Fedora(Distro): return self.run_command_with_code(['rpm', '-q', pkg], check_exit_code=False)[1] == 0 - def yum_install(self, pkg, **kwargs): - print "Attempting to install '%s' via yum" % pkg - self.run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs) - def apply_patch(self, originalfile, patchfile): self.run_command(['patch', '-N', originalfile, patchfile], check_exit_code=False) @@ -193,7 +194,7 @@ class Fedora(Distro): return if not self.check_pkg('python-virtualenv'): - self.yum_install('python-virtualenv', check_exit_code=False) + self.die("Please install 'python-virtualenv'.") super(Fedora, self).install_virtualenv() @@ -206,12 +207,13 @@ class Fedora(Distro): This can be removed when the fix is applied upstream. Nova: https://bugs.launchpad.net/nova/+bug/884915 - Upstream: https://bitbucket.org/which_linden/eventlet/issue/89 + Upstream: https://bitbucket.org/eventlet/eventlet/issue/89 + RHEL: https://bugzilla.redhat.com/958868 """ # Install "patch" program if it's not there if not self.check_pkg('patch'): - self.yum_install('patch') + self.die("Please install 'patch'.") # Apply the eventlet patch self.apply_patch(os.path.join(self.venv, 'lib', self.py_version, diff --git a/tools/patch_tox_venv.py b/tools/patch_tox_venv.py index 7100109579f..87dfac14d77 100644 --- a/tools/patch_tox_venv.py +++ b/tools/patch_tox_venv.py @@ -20,13 +20,25 @@ import sys import install_venv_common as install_venv +def first_file(file_list): + for candidate in file_list: + if os.path.exists(candidate): + return candidate + + def main(argv): root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) venv = os.environ['VIRTUAL_ENV'] - pip_requires = os.path.join(root, 'requirements.txt') - test_requires = os.path.join(root, 'test-requirements.txt') + pip_requires = first_file([ + os.path.join(root, 'requirements.txt'), + os.path.join(root, 'tools', 'pip-requires'), + ]) + test_requires = first_file([ + os.path.join(root, 'test-requirements.txt'), + os.path.join(root, 'tools', 'test-requires'), + ]) py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) project = 'cinder' install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,