diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py index 4a0251a014..47a9186586 100644 --- a/manila/tests/test_utils.py +++ b/manila/tests/test_utils.py @@ -17,7 +17,6 @@ import __builtin__ import datetime import errno -import hashlib import os import os.path import socket @@ -28,7 +27,6 @@ import mock from oslo.config import cfg from oslo.utils import timeutils import paramiko -import six import manila from manila import exception @@ -194,30 +192,6 @@ class GetFromPathTestCase(test.TestCase): class GenericUtilsTestCase(test.TestCase): - def test_hostname_unicode_sanitization(self): - hostname = u"\u7684.test.example.com" - self.assertEqual("test.example.com", - utils.sanitize_hostname(hostname)) - - def test_hostname_sanitize_periods(self): - hostname = "....test.example.com..." - self.assertEqual("test.example.com", - utils.sanitize_hostname(hostname)) - - def test_hostname_sanitize_dashes(self): - hostname = "----test.example.com---" - self.assertEqual("test.example.com", - utils.sanitize_hostname(hostname)) - - def test_hostname_sanitize_characters(self): - hostname = "(#@&$!(@*--#&91)(__=+--test-host.example!!.com-0+" - self.assertEqual("91----test-host.example.com-0", - utils.sanitize_hostname(hostname)) - - def test_hostname_translate(self): - hostname = "<}\x1fh\x10e\x08l\x02l\x05o\x12!{>" - self.assertEqual("hello", utils.sanitize_hostname(hostname)) - def test_read_cached_file(self): cache_data = {"data": 1123, "mtime": 1} with mock.patch.object(os.path, "getmtime", mock.Mock(return_value=1)): @@ -253,14 +227,6 @@ class GenericUtilsTestCase(test.TestCase): __builtin__.open.assert_called_once_with("/this/is/a/fake") os.path.getmtime.assert_called_once_with("/this/is/a/fake") - def test_generate_password(self): - password = utils.generate_password() - self.assertTrue([c for c in password if c in '0123456789']) - self.assertTrue([c for c in password - if c in 'abcdefghijklmnopqrstuvwxyz']) - self.assertTrue([c for c in password - if c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ']) - def test_read_file_as_root(self): def fake_execute(*args, **kwargs): if args[1] == 'bad': @@ -273,11 +239,6 @@ class GenericUtilsTestCase(test.TestCase): self.assertRaises(exception.FileNotFound, utils.read_file_as_root, 'bad') - def test_strcmp_const_time(self): - self.assertTrue(utils.strcmp_const_time('abc123', 'abc123')) - self.assertFalse(utils.strcmp_const_time('a', 'aaaaa')) - self.assertFalse(utils.strcmp_const_time('ABC123', 'abc123')) - def test_temporary_chown(self): def fake_execute(*args, **kwargs): if args[0] == 'chown': @@ -352,17 +313,6 @@ class GenericUtilsTestCase(test.TestCase): utils.safe_minidom_parse_string, killer_body()) - def test_xhtml_escape(self): - self.assertEqual('"foo"', utils.xhtml_escape('"foo"')) - self.assertEqual(''foo'', utils.xhtml_escape("'foo'")) - - def test_hash_file(self): - data = 'Mary had a little lamb, its fleece as white as snow' - flo = six.StringIO(data) - h1 = utils.hash_file(flo) - h2 = hashlib.sha1(data).hexdigest() - self.assertEqual(h1, h2) - def test_is_ipv6_configured0(self): fake_fd = mock.Mock() fake_fd.read.return_value = 'test' @@ -455,149 +405,6 @@ class MonkeyPatchTestCase(test.TestCase): in manila.tests.monkey_patch_example.CALLED_FUNCTION) -class AuditPeriodTest(test.TestCase): - - def setUp(self): - super(AuditPeriodTest, self).setUp() - # a fairly random time to test with - self.test_time = datetime.datetime(second=23, - minute=12, - hour=8, - day=5, - month=3, - year=2012) - self.patcher = mock.patch.object(timeutils, 'utcnow') - self.mock_utcnow = self.patcher.start() - self.mock_utcnow.return_value = self.test_time - - def tearDown(self): - self.patcher.stop() - super(AuditPeriodTest, self).tearDown() - - def test_hour(self): - begin, end = utils.last_completed_audit_period(unit='hour') - self.assertEqual(begin, - datetime.datetime(hour=7, - day=5, - month=3, - year=2012)) - self.assertEqual(end, datetime.datetime(hour=8, - day=5, - month=3, - year=2012)) - - def test_hour_with_offset_before_current(self): - begin, end = utils.last_completed_audit_period(unit='hour@10') - self.assertEqual(begin, datetime.datetime(minute=10, - hour=7, - day=5, - month=3, - year=2012)) - self.assertEqual(end, datetime.datetime(minute=10, - hour=8, - day=5, - month=3, - year=2012)) - - def test_hour_with_offset_after_current(self): - begin, end = utils.last_completed_audit_period(unit='hour@30') - self.assertEqual(begin, datetime.datetime(minute=30, - hour=6, - day=5, - month=3, - year=2012)) - self.assertEqual(end, datetime.datetime(minute=30, - hour=7, - day=5, - month=3, - year=2012)) - - def test_day(self): - begin, end = utils.last_completed_audit_period(unit='day') - self.assertEqual(begin, datetime.datetime(day=4, - month=3, - year=2012)) - self.assertEqual(end, datetime.datetime(day=5, - month=3, - year=2012)) - - def test_day_with_offset_before_current(self): - begin, end = utils.last_completed_audit_period(unit='day@6') - self.assertEqual(begin, datetime.datetime(hour=6, - day=4, - month=3, - year=2012)) - self.assertEqual(end, datetime.datetime(hour=6, - day=5, - month=3, - year=2012)) - - def test_day_with_offset_after_current(self): - begin, end = utils.last_completed_audit_period(unit='day@10') - self.assertEqual(begin, datetime.datetime(hour=10, - day=3, - month=3, - year=2012)) - self.assertEqual(end, datetime.datetime(hour=10, - day=4, - month=3, - year=2012)) - - def test_month(self): - begin, end = utils.last_completed_audit_period(unit='month') - self.assertEqual(begin, datetime.datetime(day=1, - month=2, - year=2012)) - self.assertEqual(end, datetime.datetime(day=1, - month=3, - year=2012)) - - def test_month_with_offset_before_current(self): - begin, end = utils.last_completed_audit_period(unit='month@2') - self.assertEqual(begin, datetime.datetime(day=2, - month=2, - year=2012)) - self.assertEqual(end, datetime.datetime(day=2, - month=3, - year=2012)) - - def test_month_with_offset_after_current(self): - begin, end = utils.last_completed_audit_period(unit='month@15') - self.assertEqual(begin, datetime.datetime(day=15, - month=1, - year=2012)) - self.assertEqual(end, datetime.datetime(day=15, - month=2, - year=2012)) - - def test_year(self): - begin, end = utils.last_completed_audit_period(unit='year') - self.assertEqual(begin, datetime.datetime(day=1, - month=1, - year=2011)) - self.assertEqual(end, datetime.datetime(day=1, - month=1, - year=2012)) - - def test_year_with_offset_before_current(self): - begin, end = utils.last_completed_audit_period(unit='year@2') - self.assertEqual(begin, datetime.datetime(day=1, - month=2, - year=2011)) - self.assertEqual(end, datetime.datetime(day=1, - month=2, - year=2012)) - - def test_year_with_offset_after_current(self): - begin, end = utils.last_completed_audit_period(unit='year@6') - self.assertEqual(begin, datetime.datetime(day=1, - month=6, - year=2010)) - self.assertEqual(end, datetime.datetime(day=1, - month=6, - year=2011)) - - class FakeSSHClient(object): def __init__(self): diff --git a/manila/utils.py b/manila/utils.py index 4118c34df6..2bfc9328f7 100644 --- a/manila/utils.py +++ b/manila/utils.py @@ -18,14 +18,10 @@ """Utilities and helper functions.""" import contextlib -import datetime import errno -import hashlib import inspect import os import pyclbr -import random -import re import shutil import socket import sys @@ -34,12 +30,10 @@ from xml.dom import minidom from xml.parsers import expat from xml import sax from xml.sax import expatreader -from xml.sax import saxutils from eventlet import pools import netaddr from oslo.config import cfg -from oslo.utils import excutils from oslo.utils import importutils from oslo.utils import timeutils from oslo_concurrency import processutils @@ -54,40 +48,10 @@ from manila.openstack.common import log as logging CONF = cfg.CONF LOG = logging.getLogger(__name__) -ISO_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" -PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" synchronized = lockutils.synchronized_with_prefix('manila-') -def find_config(config_path): - """Find a configuration file using the given hint. - - :param config_path: Full or relative path to the config. - :returns: Full path of the config, if it exists. - :raises: `manila.exception.ConfigNotFound` - - """ - possible_locations = [ - config_path, - os.path.join(CONF.state_path, "etc", "manila", config_path), - os.path.join(CONF.state_path, "etc", config_path), - os.path.join(CONF.state_path, config_path), - "/etc/manila/%s" % config_path, - ] - - for path in possible_locations: - if os.path.exists(path): - return os.path.abspath(path) - - raise exception.ConfigNotFound(path=os.path.abspath(config_path)) - - -def fetchfile(url, target): - LOG.debug('Fetching %s', url) - execute('curl', '--fail', url, '-o', target) - - def _get_root_helper(): return 'sudo manila-rootwrap %s' % CONF.rootwrap_config @@ -106,13 +70,6 @@ def trycmd(*args, **kwargs): return processutils.trycmd(*args, **kwargs) -def create_channel(client, width, height): - """Invoke an interactive shell session on server.""" - channel = client.invoke_shell() - channel.resize_pty(width, height) - return channel - - class SSHPool(pools.Pool): """A simple eventlet pool to hold ssh connections.""" @@ -201,182 +158,6 @@ def debug(arg): return arg -def generate_uid(topic, size=8): - characters = '01234567890abcdefghijklmnopqrstuvwxyz' - choices = [random.choice(characters) for x in xrange(size)] - return '%s-%s' % (topic, ''.join(choices)) - - -# Default symbols to use for passwords. Avoids visually confusing characters. -# ~6 bits per symbol -DEFAULT_PASSWORD_SYMBOLS = ('23456789', # Removed: 0,1 - 'ABCDEFGHJKLMNPQRSTUVWXYZ', # Removed: I, O - 'abcdefghijkmnopqrstuvwxyz') # Removed: l - - -# ~5 bits per symbol -EASIER_PASSWORD_SYMBOLS = ('23456789', # Removed: 0, 1 - 'ABCDEFGHJKLMNPQRSTUVWXYZ') # Removed: I, O - - -def last_completed_audit_period(unit=None): - """This method gives you the most recently *completed* audit period. - - arguments: - units: string, one of 'hour', 'day', 'month', 'year' - Periods normally begin at the beginning (UTC) of the - period unit (So a 'day' period begins at midnight UTC, - a 'month' unit on the 1st, a 'year' on Jan, 1) - unit string may be appended with an optional offset - like so: 'day@18' This will begin the period at 18:00 - UTC. 'month@15' starts a monthly period on the 15th, - and year@3 begins a yearly one on March 1st. - - - returns: 2 tuple of datetimes (begin, end) - The begin timestamp of this audit period is the same as the - end of the previous. - """ - if not unit: - unit = CONF.volume_usage_audit_period - - offset = 0 - if '@' in unit: - unit, offset = unit.split("@", 1) - offset = int(offset) - - rightnow = timeutils.utcnow() - if unit not in ('month', 'day', 'year', 'hour'): - raise ValueError('Time period must be hour, day, month or year') - if unit == 'month': - if offset == 0: - offset = 1 - end = datetime.datetime(day=offset, - month=rightnow.month, - year=rightnow.year) - if end >= rightnow: - year = rightnow.year - if 1 >= rightnow.month: - year -= 1 - month = 12 + (rightnow.month - 1) - else: - month = rightnow.month - 1 - end = datetime.datetime(day=offset, - month=month, - year=year) - year = end.year - if 1 >= end.month: - year -= 1 - month = 12 + (end.month - 1) - else: - month = end.month - 1 - begin = datetime.datetime(day=offset, month=month, year=year) - - elif unit == 'year': - if offset == 0: - offset = 1 - end = datetime.datetime(day=1, month=offset, year=rightnow.year) - if end >= rightnow: - end = datetime.datetime(day=1, - month=offset, - year=rightnow.year - 1) - begin = datetime.datetime(day=1, - month=offset, - year=rightnow.year - 2) - else: - begin = datetime.datetime(day=1, - month=offset, - year=rightnow.year - 1) - - elif unit == 'day': - end = datetime.datetime(hour=offset, - day=rightnow.day, - month=rightnow.month, - year=rightnow.year) - if end >= rightnow: - end = end - datetime.timedelta(days=1) - begin = end - datetime.timedelta(days=1) - - elif unit == 'hour': - end = rightnow.replace(minute=offset, second=0, microsecond=0) - if end >= rightnow: - end = end - datetime.timedelta(hours=1) - begin = end - datetime.timedelta(hours=1) - - return (begin, end) - - -def generate_password(length=20, symbolgroups=DEFAULT_PASSWORD_SYMBOLS): - """Generate a random password from the supplied symbol groups. - - At least one symbol from each group will be included. Unpredictable - results if length is less than the number of symbol groups. - - Believed to be reasonably secure (with a reasonable password length!) - - """ - r = random.SystemRandom() - - # NOTE(jerdfelt): Some password policies require at least one character - # from each group of symbols, so start off with one random character - # from each symbol group - password = [r.choice(s) for s in symbolgroups] - # If length < len(symbolgroups), the leading characters will only - # be from the first length groups. Try our best to not be predictable - # by shuffling and then truncating. - r.shuffle(password) - password = password[:length] - length -= len(password) - - # then fill with random characters from all symbol groups - symbols = ''.join(symbolgroups) - password.extend([r.choice(symbols) for _i in xrange(length)]) - - # finally shuffle to ensure first x characters aren't from a - # predictable group - r.shuffle(password) - - return ''.join(password) - - -def generate_username(length=20, symbolgroups=DEFAULT_PASSWORD_SYMBOLS): - # Use the same implementation as the password generation. - return generate_password(length, symbolgroups) - - -def last_octet(address): - return int(address.split('.')[-1]) - - -def get_my_linklocal(interface): - try: - if_str = execute('ip', '-f', 'inet6', '-o', 'addr', 'show', interface) - condition = '\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link' - links = [re.search(condition, x) for x in if_str[0].split('\n')] - address = [w.group(1) for w in links if w is not None] - if address[0] is not None: - return address[0] - else: - raise exception.Error(_('Link Local address is not found.:%s') - % if_str) - except Exception as ex: - raise exception.Error(_("Couldn't get Link Local IP of %(interface)s" - " :%(ex)s") % - {"interface": interface, "ex": ex}) - - -def parse_mailmap(mailmap='.mailmap'): - mapping = {} - if os.path.exists(mailmap): - fp = open(mailmap, 'r') - for l in fp: - l = l.strip() - if not l.startswith('#') and ' ' in l: - canonical_email, alias = l.split(' ') - mapping[alias.lower()] = canonical_email.lower() - return mapping - - class LazyPluggable(object): """A pluggable backend loaded lazily based on some value.""" @@ -448,26 +229,6 @@ def safe_minidom_parse_string(xml_string): raise expat.ExpatError() -def xhtml_escape(value): - """Escapes a string so it is valid within XML or XHTML. - - """ - return saxutils.escape(value, {'"': '"', "'": '''}) - - -def utf8(value): - """Try to turn a string into utf-8 if possible. - - Code is directly from the utf8 function in - http://github.com/facebook/tornado/blob/master/tornado/escape.py - - """ - if isinstance(value, unicode): - return value.encode('utf-8') - assert isinstance(value, str) - return value - - def delete_if_exists(pathname): """Delete a file, but ignore file not found error.""" @@ -531,33 +292,6 @@ def get_from_path(items, path): return get_from_path(results, remainder) -def is_valid_boolstr(val): - """Check if the provided string is a valid bool string or not.""" - val = str(val).lower() - return (val == 'true' or val == 'false' or - val == 'yes' or val == 'no' or - val == 'y' or val == 'n' or - val == '1' or val == '0') - - -def is_valid_ipv4(address): - """Validate IPv4 address. - - Valid the address strictly as per format xxx.xxx.xxx.xxx. - where xxx is a value between 0 and 255. - """ - parts = address.split(".") - if len(parts) != 4: - return False - for item in parts: - try: - if not 0 <= int(item) <= 255: - return False - except ValueError: - return False - return True - - def is_ipv6_configured(): """Check if system contain IPv6 capable network interface. @@ -641,34 +375,6 @@ def monkey_patch(): decorator("%s.%s" % (module, key), func)) -def make_dev_path(dev, partition=None, base='/dev'): - """Return a path to a particular device. - - >>> make_dev_path('xvdc') - /dev/xvdc - - >>> make_dev_path('xvdc', 1) - /dev/xvdc1 - """ - path = os.path.join(base, dev) - if partition: - path += str(partition) - return path - - -def sanitize_hostname(hostname): - """Return a hostname which conforms to RFC-952 and RFC-1123 specs.""" - if isinstance(hostname, unicode): - hostname = hostname.encode('latin-1', 'ignore') - - hostname = re.sub('[ _]', '-', hostname) - hostname = re.sub('[^\w.-]+', '', hostname) - hostname = hostname.lower() - hostname = hostname.strip('.-') - - return hostname - - def read_cached_file(filename, cache_info, reload_func=None): """Read from a file if it has been modified. @@ -701,43 +407,6 @@ def file_open(*args, **kwargs): return file(*args, **kwargs) -def hash_file(file_like_object): - """Generate a hash for the contents of a file.""" - checksum = hashlib.sha1() - any(map(checksum.update, iter(lambda: file_like_object.read(32768), ''))) - return checksum.hexdigest() - - -@contextlib.contextmanager -def temporary_mutation(obj, **kwargs): - """Temporarily set the attr on a particular object. - - Temporarily set the attr on a particular object to a given value then - revert when finished. - - One use of this is to temporarily set the read_deleted flag on a context - object: - - with temporary_mutation(context, read_deleted="yes"): - do_something_that_needed_deleted_objects() - """ - NOT_PRESENT = object() - - old_values = {} - for attr, new_value in kwargs.items(): - old_values[attr] = getattr(obj, attr, NOT_PRESENT) - setattr(obj, attr, new_value) - - try: - yield - finally: - for attr, old_value in old_values.items(): - if old_value is NOT_PRESENT: - del obj[attr] - else: - setattr(obj, attr, old_value) - - def service_is_up(service): """Check whether a service is up based on last heartbeat.""" last_heartbeat = service['updated_at'] or service['created_at'] @@ -746,21 +415,6 @@ def service_is_up(service): return abs(elapsed) <= CONF.service_down_time -def generate_mac_address(): - """Generate an Ethernet MAC address.""" - # NOTE(vish): We would prefer to use 0xfe here to ensure that linux - # bridge mac addresses don't change, but it appears to - # conflict with libvirt, so we use the next highest octet - # that has the unicast and locally administered bits set - # properly: 0xfa. - # Discussion: https://bugs.launchpad.net/manila/+bug/921838 - mac = [0xfa, 0x16, 0x3e, - random.randint(0x00, 0x7f), - random.randint(0x00, 0xff), - random.randint(0x00, 0xff)] - return ':'.join(map(lambda x: "%02x" % x, mac)) - - def read_file_as_root(file_path): """Secure helper to read file as root.""" try: @@ -802,26 +456,6 @@ def tempdir(**kwargs): LOG.debug('Could not remove tmpdir: %s', six.text_type(e)) -def strcmp_const_time(s1, s2): - """Constant-time string comparison. - - :params s1: the first string - :params s2: the second string - - :return: True if the strings are equal. - - This function takes two strings and compares them. It is intended to be - used when doing a comparison for authentication purposes to help guard - against timing attacks. - """ - if len(s1) != len(s2): - return False - result = 0 - for (a, b) in zip(s1, s2): - result |= ord(a) ^ ord(b) - return result == 0 - - def walk_class_hierarchy(clazz, encountered=None): """Walk class hierarchy, yielding most derived classes first.""" if not encountered: @@ -835,34 +469,6 @@ def walk_class_hierarchy(clazz, encountered=None): yield subclass -class UndoManager(object): - """Provides a mechanism to facilitate rolling back a series of actions. - - This can be used when an exception is raised. - """ - def __init__(self): - self.undo_stack = [] - - def undo_with(self, undo_func): - self.undo_stack.append(undo_func) - - def _rollback(self): - for undo_func in reversed(self.undo_stack): - undo_func() - - def rollback_and_reraise(self, msg=None, **kwargs): - """Rollback a series of actions then re-raise the exception. - - .. note:: (sirp) This should only be called within an - exception handler. - """ - with excutils.save_and_reraise_exception(): - if msg: - LOG.exception(msg, **kwargs) - - self._rollback() - - def ensure_tree(path): """Create a directory (and any ancestor directories required) @@ -878,42 +484,6 @@ def ensure_tree(path): raise -def to_bytes(text, default=0): - """Try to turn a string into a number of bytes. - - Looks at the last characters of the text to determine what - conversion is needed to turn the input text into a byte number. - - Supports: B/b, K/k, M/m, G/g, T/t (or the same with b/B on the end). - """ - BYTE_MULTIPLIERS = { - '': 1, - 't': 1024 ** 4, - 'g': 1024 ** 3, - 'm': 1024 ** 2, - 'k': 1024, - } - - # Take off everything not number 'like' (which should leave - # only the byte 'identifier' left) - mult_key_org = text.lstrip('-1234567890') - mult_key = mult_key_org.lower() - mult_key_len = len(mult_key) - if mult_key.endswith("b"): - mult_key = mult_key[0:-1] - try: - multiplier = BYTE_MULTIPLIERS[mult_key] - if mult_key_len: - # Empty cases shouldn't cause text[0:-0] - text = text[0:-mult_key_len] - return int(text) * multiplier - except KeyError: - msg = _('Unknown byte multiplier: %s') % mult_key_org - raise TypeError(msg) - except ValueError: - return default - - def cidr_to_netmask(cidr): """Convert cidr to netmask.""" try: