From 9411a24ba7b5a1380ec8a2aa13d4aa92988d2ff9 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Mon, 7 Oct 2013 12:10:31 +0000 Subject: [PATCH] Revert "Refactor common/utils methods to common/ondisk" This reverts commit 7760f41c3ce436cb23b4b8425db3749a3da33d32 Change-Id: I95e57a2563784a8cd5e995cc826afeac0eadbe62 Signed-off-by: Peter Portante --- bin/swift-get-nodes | 2 +- bin/swift-object-info | 2 +- doc/source/misc.rst | 9 - swift/account/auditor.py | 5 +- swift/account/backend.py | 3 +- swift/account/server.py | 5 +- swift/account/utils.py | 3 +- swift/common/daemon.py | 3 +- swift/common/db.py | 5 +- swift/common/db_replicator.py | 7 +- swift/common/direct_client.py | 3 +- swift/common/ondisk.py | 184 ------------------ swift/common/request_helpers.py | 3 +- swift/common/ring/ring.py | 5 +- swift/common/utils.py | 163 +++++++++++++++- swift/common/wsgi.py | 3 +- swift/container/auditor.py | 5 +- swift/container/backend.py | 3 +- swift/container/server.py | 5 +- swift/container/sync.py | 5 +- swift/obj/auditor.py | 5 +- swift/obj/diskfile.py | 7 +- swift/obj/server.py | 6 +- swift/proxy/controllers/base.py | 6 +- swift/proxy/controllers/obj.py | 8 +- test/probe/test_container_failures.py | 3 +- test/probe/test_object_failures.py | 3 +- test/unit/account/test_backend.py | 2 +- test/unit/account/test_reaper.py | 2 +- test/unit/account/test_server.py | 3 +- .../common/middleware/test_list_endpoints.py | 6 +- test/unit/common/ring/test_ring.py | 18 +- test/unit/common/test_daemon.py | 6 +- test/unit/common/test_db.py | 2 +- test/unit/common/test_db_replicator.py | 2 +- test/unit/common/test_ondisk.py | 139 ------------- test/unit/common/test_utils.py | 104 ++++++++++ test/unit/container/test_backend.py | 2 +- test/unit/container/test_replicator.py | 2 +- test/unit/container/test_server.py | 3 +- test/unit/container/test_updater.py | 2 +- test/unit/obj/test_auditor.py | 3 +- test/unit/obj/test_diskfile.py | 9 +- test/unit/obj/test_replicator.py | 9 +- test/unit/obj/test_server.py | 23 ++- test/unit/obj/test_updater.py | 10 +- test/unit/proxy/test_server.py | 6 +- 47 files changed, 359 insertions(+), 455 deletions(-) delete mode 100644 swift/common/ondisk.py delete mode 100644 test/unit/common/test_ondisk.py diff --git a/bin/swift-get-nodes b/bin/swift-get-nodes index 0f7a742c82..36cfb7ecaa 100755 --- a/bin/swift-get-nodes +++ b/bin/swift-get-nodes @@ -19,7 +19,7 @@ import sys import urllib from swift.common.ring import Ring -from swift.common.ondisk import hash_path +from swift.common.utils import hash_path parser = optparse.OptionParser() diff --git a/bin/swift-object-info b/bin/swift-object-info index b09104cc46..da09795db7 100755 --- a/bin/swift-object-info +++ b/bin/swift-object-info @@ -20,7 +20,7 @@ from hashlib import md5 from swift.common.ring import Ring from swift.obj.diskfile import read_metadata -from swift.common.ondisk import hash_path +from swift.common.utils import hash_path if __name__ == '__main__': if len(sys.argv) <= 1: diff --git a/doc/source/misc.rst b/doc/source/misc.rst index e8921173f1..e42a91193d 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -33,15 +33,6 @@ Utils :members: :show-inheritance: -.. _ondisk: - -OnDisk -====== - -.. automodule:: swift.common.ondisk - :members: - :show-inheritance: - .. _common_tempauth: TempAuth diff --git a/swift/account/auditor.py b/swift/account/auditor.py index e60e58f2c1..3410089d6d 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -21,9 +21,8 @@ from random import random import swift.common.db from swift.account import server as account_server from swift.account.backend import AccountBroker -from swift.common.utils import get_logger, config_true_value, \ - dump_recon_cache, ratelimit_sleep -from swift.common.ondisk import audit_location_generator +from swift.common.utils import get_logger, audit_location_generator, \ + config_true_value, dump_recon_cache, ratelimit_sleep from swift.common.daemon import Daemon from eventlet import Timeout diff --git a/swift/account/backend.py b/swift/account/backend.py index c5d661a119..96c7556b9b 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -25,8 +25,7 @@ import errno import sqlite3 -from swift.common.utils import lock_parent_directory -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp, lock_parent_directory from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode diff --git a/swift/account/server.py b/swift/account/server.py index 5ed982ec9a..53770c44ed 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -28,10 +28,9 @@ from swift.account.utils import account_listing_response from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path -from swift.common.utils import get_logger, public, config_true_value, \ +from swift.common.utils import get_logger, hash_path, public, \ + normalize_timestamp, storage_directory, config_true_value, \ json, timing_stats, replication -from swift.common.ondisk import hash_path, normalize_timestamp, \ - storage_directory from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc diff --git a/swift/account/utils.py b/swift/account/utils.py index c40dbc7ed0..9854fae41f 100644 --- a/swift/account/utils.py +++ b/swift/account/utils.py @@ -17,8 +17,7 @@ import time from xml.sax import saxutils from swift.common.swob import HTTPOk, HTTPNoContent -from swift.common.utils import json -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import json, normalize_timestamp class FakeAccountBroker(object): diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 11b86a4304..7b2ea93c03 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -22,7 +22,6 @@ from re import sub import eventlet.debug from swift.common import utils -from swift.common import ondisk class Daemon(object): @@ -42,7 +41,7 @@ class Daemon(object): def run(self, once=False, **kwargs): """Run the daemon""" - ondisk.validate_configuration() + utils.validate_configuration() utils.drop_privileges(self.conf.get('user', 'swift')) utils.capture_stdio(self.logger, **kwargs) diff --git a/swift/common/db.py b/swift/common/db.py index 3e14d176b3..aca0cc6d8f 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -30,9 +30,8 @@ from tempfile import mkstemp from eventlet import sleep, Timeout import sqlite3 -from swift.common.utils import json, renamer, mkdirs, lock_parent_directory, \ - fallocate -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import json, normalize_timestamp, renamer, \ + mkdirs, lock_parent_directory, fallocate from swift.common.exceptions import LockTimeout diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 4fba675a8a..1ac38ac5e0 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -30,10 +30,9 @@ import simplejson import swift.common.db from swift.common.direct_client import quote -from swift.common.utils import get_logger, whataremyips, renamer, mkdirs, \ - lock_parent_directory, config_true_value, unlink_older_than, \ - dump_recon_cache, rsync_ip -from swift.common.ondisk import storage_directory +from swift.common.utils import get_logger, whataremyips, storage_directory, \ + renamer, mkdirs, lock_parent_directory, config_true_value, \ + unlink_older_than, dump_recon_cache, rsync_ip from swift.common import ring from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE from swift.common.bufferedhttp import BufferedHTTPConnection diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index f7a954dcd3..08dde9b524 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -27,8 +27,7 @@ from eventlet import sleep, Timeout from swift.common.bufferedhttp import http_connect from swiftclient import ClientException, json_loads -from swift.common.utils import FileLikeIter -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp, FileLikeIter from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \ is_success, is_server_error from swift.common.swob import HeaderKeyDict diff --git a/swift/common/ondisk.py b/swift/common/ondisk.py deleted file mode 100644 index 86b1d82279..0000000000 --- a/swift/common/ondisk.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright (c) 2010-2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Methods & Attributes for shared 'on-disk' data layouts.""" - -import os -import sys -import errno - -from hashlib import md5 -from random import shuffle -from ConfigParser import ConfigParser, NoSectionError, NoOptionError - -from swift import gettext_ as _ -from swift.common.utils import listdir, quote - -# Used by hash_path to offer a bit more security when generating hashes for -# paths. It simply appends this value to all paths; guessing the hash a path -# will end up with would also require knowing this suffix. -_hash_conf = ConfigParser() -HASH_PATH_SUFFIX = '' -HASH_PATH_PREFIX = '' -if _hash_conf.read('/etc/swift/swift.conf'): - try: - HASH_PATH_SUFFIX = _hash_conf.get('swift-hash', - 'swift_hash_path_suffix') - except (NoSectionError, NoOptionError): - pass - try: - HASH_PATH_PREFIX = _hash_conf.get('swift-hash', - 'swift_hash_path_prefix') - except (NoSectionError, NoOptionError): - pass - - -def validate_configuration(): - if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX: - sys.exit("Error: [swift-hash]: both swift_hash_path_suffix " - "and swift_hash_path_prefix are missing " - "from /etc/swift/swift.conf") - - -def hash_path(account, container=None, object=None, raw_digest=False): - """ - Get the canonical hash for an account/container/object - - :param account: Account - :param container: Container - :param object: Object - :param raw_digest: If True, return the raw version rather than a hex digest - :returns: hash string - """ - if object and not container: - raise ValueError('container is required if object is provided') - paths = [account] - if container: - paths.append(container) - if object: - paths.append(object) - if raw_digest: - return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) - + HASH_PATH_SUFFIX).digest() - else: - return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) - + HASH_PATH_SUFFIX).hexdigest() - - -def normalize_timestamp(timestamp): - """ - Format a timestamp (string or numeric) into a standardized - xxxxxxxxxx.xxxxx (10.5) format. - - Note that timestamps using values greater than or equal to November 20th, - 2286 at 17:46 UTC will use 11 digits to represent the number of - seconds. - - :param timestamp: unix timestamp - :returns: normalized timestamp as a string - """ - return "%016.05f" % (float(timestamp)) - - -def validate_device_partition(device, partition): - """ - Validate that a device and a partition are valid and won't lead to - directory traversal when used. - - :param device: device to validate - :param partition: partition to validate - :raises: ValueError if given an invalid device or partition - """ - invalid_device = False - invalid_partition = False - if not device or '/' in device or device in ['.', '..']: - invalid_device = True - if not partition or '/' in partition or partition in ['.', '..']: - invalid_partition = True - - if invalid_device: - raise ValueError('Invalid device: %s' % quote(device or '')) - elif invalid_partition: - raise ValueError('Invalid partition: %s' % quote(partition or '')) - - -def storage_directory(datadir, partition, name_hash): - """ - Get the storage directory - - :param datadir: Base data directory - :param partition: Partition - :param name_hash: Account, container or object name hash - :returns: Storage directory - """ - return os.path.join(datadir, str(partition), name_hash[-3:], name_hash) - - -def audit_location_generator(devices, datadir, suffix='', - mount_check=True, logger=None): - ''' - Given a devices path and a data directory, yield (path, device, - partition) for all files in that directory - - :param devices: parent directory of the devices to be audited - :param datadir: a directory located under self.devices. This should be - one of the DATADIR constants defined in the account, - container, and object servers. - :param suffix: path name suffix required for all names returned - :param mount_check: Flag to check if a mount check should be performed - on devices - :param logger: a logger object - ''' - device_dir = listdir(devices) - # randomize devices in case of process restart before sweep completed - shuffle(device_dir) - for device in device_dir: - if mount_check and not \ - os.path.ismount(os.path.join(devices, device)): - if logger: - logger.debug( - _('Skipping %s as it is not mounted'), device) - continue - datadir_path = os.path.join(devices, device, datadir) - partitions = listdir(datadir_path) - for partition in partitions: - part_path = os.path.join(datadir_path, partition) - try: - suffixes = listdir(part_path) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for asuffix in suffixes: - suff_path = os.path.join(part_path, asuffix) - try: - hashes = listdir(suff_path) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for hsh in hashes: - hash_path = os.path.join(suff_path, hsh) - try: - files = sorted(listdir(hash_path), reverse=True) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for fname in files: - if suffix and not fname.endswith(suffix): - continue - path = os.path.join(hash_path, fname) - yield path, device, partition diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index 06ad4af09b..c5584d5796 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -22,8 +22,7 @@ from swob in here without creating circular imports. from swift.common.constraints import FORMAT2CONTENT_TYPE from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable -from swift.common.ondisk import validate_device_partition -from swift.common.utils import split_path +from swift.common.utils import split_path, validate_device_partition from urllib import unquote diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 1d16e1b9fc..8f0d2835d1 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -25,8 +25,7 @@ from io import BufferedReader from hashlib import md5 from itertools import chain -from swift.common.utils import json -from swift.common.ondisk import hash_path, validate_configuration +from swift.common.utils import hash_path, validate_configuration, json from swift.common.ring.utils import tiers_for_dev @@ -131,7 +130,7 @@ class Ring(object): """ def __init__(self, serialized_path, reload_time=15, ring_name=None): - # Can't use the ring unless the on-disk configuration is valid + # can't use the ring unless HASH_PATH_SUFFIX is set validate_configuration() if ring_name: self.serialized_path = os.path.join(serialized_path, diff --git a/swift/common/utils.py b/swift/common/utils.py index 27bb82b543..0fbfc9f8dd 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -26,12 +26,14 @@ import threading as stdlib_threading import time import uuid import functools -from random import random +from hashlib import md5 +from random import random, shuffle from urllib import quote as _quote from contextlib import contextmanager, closing import ctypes import ctypes.util -from ConfigParser import ConfigParser, RawConfigParser +from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \ + RawConfigParser from optparse import OptionParser from Queue import Queue, Empty from tempfile import mkstemp, NamedTemporaryFile @@ -78,6 +80,24 @@ _posix_fadvise = None # available being at or below this amount, in bytes. FALLOCATE_RESERVE = 0 +# Used by hash_path to offer a bit more security when generating hashes for +# paths. It simply appends this value to all paths; guessing the hash a path +# will end up with would also require knowing this suffix. +hash_conf = ConfigParser() +HASH_PATH_SUFFIX = '' +HASH_PATH_PREFIX = '' +if hash_conf.read('/etc/swift/swift.conf'): + try: + HASH_PATH_SUFFIX = hash_conf.get('swift-hash', + 'swift_hash_path_suffix') + except (NoSectionError, NoOptionError): + pass + try: + HASH_PATH_PREFIX = hash_conf.get('swift-hash', + 'swift_hash_path_prefix') + except (NoSectionError, NoOptionError): + pass + def backward(f, blocksize=4096): """ @@ -144,6 +164,13 @@ def noop_libc_function(*args): return 0 +def validate_configuration(): + if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX: + sys.exit("Error: [swift-hash]: both swift_hash_path_suffix " + "and swift_hash_path_prefix are missing " + "from /etc/swift/swift.conf") + + def load_libc_function(func_name, log_error=True): """ Attempt to find the function in libc, otherwise return a no-op func. @@ -395,6 +422,21 @@ def drop_buffer_cache(fd, offset, length): % (fd, offset, length, ret)) +def normalize_timestamp(timestamp): + """ + Format a timestamp (string or numeric) into a standardized + xxxxxxxxxx.xxxxx (10.5) format. + + Note that timestamps using values greater than or equal to November 20th, + 2286 at 17:46 UTC will use 11 digits to represent the number of + seconds. + + :param timestamp: unix timestamp + :returns: normalized timestamp as a string + """ + return "%016.05f" % (float(timestamp)) + + def mkdirs(path): """ Ensures the path is a directory or makes it if not. Errors if the path @@ -473,6 +515,28 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): return segs +def validate_device_partition(device, partition): + """ + Validate that a device and a partition are valid and won't lead to + directory traversal when used. + + :param device: device to validate + :param partition: partition to validate + :raises: ValueError if given an invalid device or partition + """ + invalid_device = False + invalid_partition = False + if not device or '/' in device or device in ['.', '..']: + invalid_device = True + if not partition or '/' in partition or partition in ['.', '..']: + invalid_partition = True + + if invalid_device: + raise ValueError('Invalid device: %s' % quote(device or '')) + elif invalid_partition: + raise ValueError('Invalid partition: %s' % quote(partition or '')) + + class GreenthreadSafeIterator(object): """ Wrap an iterator to ensure that only one greenthread is inside its next() @@ -1076,6 +1140,43 @@ def whataremyips(): return addresses +def storage_directory(datadir, partition, name_hash): + """ + Get the storage directory + + :param datadir: Base data directory + :param partition: Partition + :param name_hash: Account, container or object name hash + :returns: Storage directory + """ + return os.path.join(datadir, str(partition), name_hash[-3:], name_hash) + + +def hash_path(account, container=None, object=None, raw_digest=False): + """ + Get the canonical hash for an account/container/object + + :param account: Account + :param container: Container + :param object: Object + :param raw_digest: If True, return the raw version rather than a hex digest + :returns: hash string + """ + if object and not container: + raise ValueError('container is required if object is provided') + paths = [account] + if container: + paths.append(container) + if object: + paths.append(object) + if raw_digest: + return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) + + HASH_PATH_SUFFIX).digest() + else: + return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) + + HASH_PATH_SUFFIX).hexdigest() + + @contextmanager def lock_path(directory, timeout=10): """ @@ -1390,6 +1491,64 @@ def remove_file(path): pass +def audit_location_generator(devices, datadir, suffix='', + mount_check=True, logger=None): + ''' + Given a devices path and a data directory, yield (path, device, + partition) for all files in that directory + + :param devices: parent directory of the devices to be audited + :param datadir: a directory located under self.devices. This should be + one of the DATADIR constants defined in the account, + container, and object servers. + :param suffix: path name suffix required for all names returned + :param mount_check: Flag to check if a mount check should be performed + on devices + :param logger: a logger object + ''' + device_dir = listdir(devices) + # randomize devices in case of process restart before sweep completed + shuffle(device_dir) + for device in device_dir: + if mount_check and not \ + os.path.ismount(os.path.join(devices, device)): + if logger: + logger.debug( + _('Skipping %s as it is not mounted'), device) + continue + datadir_path = os.path.join(devices, device, datadir) + partitions = listdir(datadir_path) + for partition in partitions: + part_path = os.path.join(datadir_path, partition) + try: + suffixes = listdir(part_path) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for asuffix in suffixes: + suff_path = os.path.join(part_path, asuffix) + try: + hashes = listdir(suff_path) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for hsh in hashes: + hash_path = os.path.join(suff_path, hsh) + try: + files = sorted(listdir(hash_path), reverse=True) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for fname in files: + if suffix and not fname.endswith(suffix): + continue + path = os.path.join(hash_path, fname) + yield path, device, partition + + def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5): ''' Will eventlet.sleep() for the appropriate time so that the max_rate diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index a33b8aa5ca..1118abbdd5 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -35,8 +35,7 @@ from swift.common import utils from swift.common.swob import Request from swift.common.utils import capture_stdio, disable_fallocate, \ drop_privileges, get_logger, NullLogger, config_true_value, \ - get_hub, config_auto_int_value -from swift.common.ondisk import validate_configuration + validate_configuration, get_hub, config_auto_int_value try: import multiprocessing diff --git a/swift/container/auditor.py b/swift/container/auditor.py index fc8f1c6641..f0d414fee5 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -23,9 +23,8 @@ from eventlet import Timeout import swift.common.db from swift.container import server as container_server from swift.container.backend import ContainerBroker -from swift.common.utils import get_logger, config_true_value, \ - dump_recon_cache, ratelimit_sleep -from swift.common.ondisk import audit_location_generator +from swift.common.utils import get_logger, audit_location_generator, \ + config_true_value, dump_recon_cache, ratelimit_sleep from swift.common.daemon import Daemon diff --git a/swift/container/backend.py b/swift/container/backend.py index 5332333fb2..a2b36fbb0e 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -25,8 +25,7 @@ import errno import sqlite3 -from swift.common.utils import lock_parent_directory -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp, lock_parent_directory from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode diff --git a/swift/container/server.py b/swift/container/server.py index 487556d06c..4810b9bfaf 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -29,11 +29,10 @@ from swift.container.backend import ContainerBroker from swift.common.db import DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path -from swift.common.utils import get_logger, public, validate_sync_to, \ +from swift.common.utils import get_logger, hash_path, public, \ + normalize_timestamp, storage_directory, validate_sync_to, \ config_true_value, json, timing_stats, replication, \ override_bytes_from_content_type -from swift.common.ondisk import hash_path, normalize_timestamp, \ - storage_directory from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.bufferedhttp import http_connect diff --git a/swift/container/sync.py b/swift/container/sync.py index f90d24f21e..9594c852fb 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -27,9 +27,8 @@ from swiftclient import ClientException, delete_object, put_object, \ from swift.container.backend import ContainerBroker from swift.common.direct_client import direct_get_object from swift.common.ring import Ring -from swift.common.utils import get_logger, config_true_value, \ - validate_sync_to, whataremyips, FileLikeIter -from swift.common.ondisk import audit_location_generator, hash_path +from swift.common.utils import audit_location_generator, get_logger, \ + hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 67b9f8d9e7..2a491a87f0 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -21,9 +21,8 @@ from eventlet import Timeout from swift.obj import diskfile from swift.obj import server as object_server -from swift.common.utils import get_logger, ratelimit_sleep, \ - config_true_value, dump_recon_cache, list_from_csv, json -from swift.common.ondisk import audit_location_generator +from swift.common.utils import get_logger, audit_location_generator, \ + ratelimit_sleep, config_true_value, dump_recon_cache, list_from_csv, json from swift.common.exceptions import AuditException, DiskFileError, \ DiskFileNotExist from swift.common.daemon import Daemon diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index 47e342c077..656c1797c9 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -33,10 +33,9 @@ from eventlet import Timeout from swift import gettext_ as _ from swift.common.constraints import check_mount -from swift.common.utils import mkdirs, renamer, fallocate, fsync, fdatasync, \ - drop_buffer_cache, ThreadPool, lock_path, write_pickle -from swift.common.ondisk import hash_path, normalize_timestamp, \ - storage_directory +from swift.common.utils import mkdirs, normalize_timestamp, \ + storage_directory, hash_path, renamer, fallocate, fsync, \ + fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle from swift.common.exceptions import DiskFileError, DiskFileNotExist, \ DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \ PathNotDir, DiskFileNotOpenError diff --git a/swift/obj/server.py b/swift/obj/server.py index 13d106fafc..ab70ff21cf 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -27,9 +27,9 @@ from hashlib import md5 from eventlet import sleep, Timeout -from swift.common.utils import mkdirs, public, get_logger, write_pickle, \ - config_true_value, timing_stats, ThreadPool, replication -from swift.common.ondisk import normalize_timestamp, hash_path +from swift.common.utils import mkdirs, normalize_timestamp, public, \ + hash_path, get_logger, write_pickle, config_true_value, timing_stats, \ + ThreadPool, replication from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 55b09f29ac..ed3f63acde 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -37,9 +37,9 @@ from eventlet.queue import Queue, Empty, Full from eventlet.timeout import Timeout from swift.common.wsgi import make_pre_authed_env -from swift.common.utils import config_true_value, public, split_path, \ - list_from_csv, GreenthreadSafeIterator, quorum_size -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp, config_true_value, \ + public, split_path, list_from_csv, GreenthreadSafeIterator, \ + quorum_size from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout from swift.common.http import is_informational, is_success, is_redirection, \ diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 5523283e4d..db8c188b1a 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -38,10 +38,10 @@ from eventlet import sleep, GreenPile from eventlet.queue import Queue from eventlet.timeout import Timeout -from swift.common.utils import ContextPool, config_true_value, public, json, \ - csv_append, GreenthreadSafeIterator, quorum_size, split_path, \ - override_bytes_from_content_type, get_valid_utf8_str -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import ContextPool, normalize_timestamp, \ + config_true_value, public, json, csv_append, GreenthreadSafeIterator, \ + quorum_size, split_path, override_bytes_from_content_type, \ + get_valid_utf8_str from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index a263525e9d..c9df7ff660 100755 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -25,8 +25,7 @@ from sqlite3 import connect from swiftclient import client from swift.common import direct_client -from swift.common.ondisk import hash_path -from swift.common.utils import readconf +from swift.common.utils import hash_path, readconf from test.probe.common import get_to_final_state, kill_nonprimary_server, \ kill_server, kill_servers, reset_environment, start_server diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index b9f8b6af80..fe5ee17eb6 100755 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -22,8 +22,7 @@ from uuid import uuid4 from swiftclient import client from swift.common import direct_client -from swift.common.ondisk import hash_path -from swift.common.utils import readconf +from swift.common.utils import hash_path, readconf from swift.obj.diskfile import write_metadata, read_metadata from test.probe.common import kill_servers, reset_environment diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py index 19fa7aea87..4decfbde5f 100644 --- a/test/unit/account/test_backend.py +++ b/test/unit/account/test_backend.py @@ -22,7 +22,7 @@ from time import sleep, time from uuid import uuid4 from swift.account.backend import AccountBroker -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp class TestAccountBroker(unittest.TestCase): diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index 21e6b64755..ee0700eb7b 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -25,7 +25,7 @@ from contextlib import nested from swift.account import reaper from swift.account.server import DATADIR -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp from swift.common.direct_client import ClientException diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index b3192ea832..d224e1b8e6 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -25,8 +25,7 @@ import xml.dom.minidom from swift.common.swob import Request from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT -from swift.common.utils import replication, public -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp, replication, public class TestAccountController(unittest.TestCase): diff --git a/test/unit/common/middleware/test_list_endpoints.py b/test/unit/common/middleware/test_list_endpoints.py index c3ed3c8824..1f98cc72c9 100644 --- a/test/unit/common/middleware/test_list_endpoints.py +++ b/test/unit/common/middleware/test_list_endpoints.py @@ -18,7 +18,7 @@ import unittest from shutil import rmtree import os -from swift.common import ring, ondisk +from swift.common import ring, utils from swift.common.utils import json from swift.common.swob import Request, Response from swift.common.middleware import list_endpoints @@ -35,8 +35,8 @@ def start_response(*args): class TestListEndpoints(unittest.TestCase): def setUp(self): - ondisk.HASH_PATH_SUFFIX = 'endcap' - ondisk.HASH_PATH_PREFIX = '' + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'ring') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index 92410e5f60..42852042e3 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -23,7 +23,7 @@ from gzip import GzipFile from shutil import rmtree from time import sleep, time -from swift.common import ring, ondisk +from swift.common import ring, utils class TestRingData(unittest.TestCase): @@ -99,8 +99,8 @@ class TestRingData(unittest.TestCase): class TestRing(unittest.TestCase): def setUp(self): - ondisk.HASH_PATH_SUFFIX = 'endcap' - ondisk.HASH_PATH_PREFIX = '' + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'ring') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) @@ -146,15 +146,15 @@ class TestRing(unittest.TestCase): self.assertEquals(self.ring.reload_time, self.intended_reload_time) self.assertEquals(self.ring.serialized_path, self.testgz) # test invalid endcap - _orig_hash_path_suffix = ondisk.HASH_PATH_SUFFIX - _orig_hash_path_prefix = ondisk.HASH_PATH_PREFIX + _orig_hash_path_suffix = utils.HASH_PATH_SUFFIX + _orig_hash_path_prefix = utils.HASH_PATH_PREFIX try: - ondisk.HASH_PATH_SUFFIX = '' - ondisk.HASH_PATH_PREFIX = '' + utils.HASH_PATH_SUFFIX = '' + utils.HASH_PATH_PREFIX = '' self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever') finally: - ondisk.HASH_PATH_SUFFIX = _orig_hash_path_suffix - ondisk.HASH_PATH_PREFIX = _orig_hash_path_prefix + utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix + utils.HASH_PATH_PREFIX = _orig_hash_path_prefix def test_has_changed(self): self.assertEquals(self.ring.has_changed(), False) diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index 47d4233d02..e20fcde62e 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -23,7 +23,7 @@ from StringIO import StringIO from test.unit import tmpfile from mock import patch -from swift.common import daemon, utils, ondisk +from swift.common import daemon, utils class MyDaemon(daemon.Daemon): @@ -63,8 +63,8 @@ class TestDaemon(unittest.TestCase): class TestRunDaemon(unittest.TestCase): def setUp(self): - ondisk.HASH_PATH_SUFFIX = 'endcap' - ondisk.HASH_PATH_PREFIX = 'startcap' + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = 'startcap' utils.drop_privileges = lambda *args: None utils.capture_stdio = lambda *args: None diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index f31809f8a5..d944c30a6d 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -28,7 +28,7 @@ from mock import patch import swift.common.db from swift.common.db import chexor, dict_factory, get_db_connection, \ DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp from swift.common.exceptions import LockTimeout diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index a0acb16e65..a9a41f7084 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -26,7 +26,7 @@ import mock import simplejson from swift.common import db_replicator -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp from swift.container import server as container_server from swift.common.exceptions import DriveNotMounted diff --git a/test/unit/common/test_ondisk.py b/test/unit/common/test_ondisk.py deleted file mode 100644 index 5592f911e8..0000000000 --- a/test/unit/common/test_ondisk.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (c) 2010-2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for swift.common.ondisk""" - -from __future__ import with_statement -from test.unit import temptree - -import os - -import unittest - -from swift.common import ondisk - - -class TestOndisk(unittest.TestCase): - """Tests for swift.common.ondisk""" - - def setUp(self): - ondisk.HASH_PATH_SUFFIX = 'endcap' - ondisk.HASH_PATH_PREFIX = 'startcap' - - def test_normalize_timestamp(self): - # Test swift.common.ondisk.normalize_timestamp - self.assertEquals(ondisk.normalize_timestamp('1253327593.48174'), - "1253327593.48174") - self.assertEquals(ondisk.normalize_timestamp(1253327593.48174), - "1253327593.48174") - self.assertEquals(ondisk.normalize_timestamp('1253327593.48'), - "1253327593.48000") - self.assertEquals(ondisk.normalize_timestamp(1253327593.48), - "1253327593.48000") - self.assertEquals(ondisk.normalize_timestamp('253327593.48'), - "0253327593.48000") - self.assertEquals(ondisk.normalize_timestamp(253327593.48), - "0253327593.48000") - self.assertEquals(ondisk.normalize_timestamp('1253327593'), - "1253327593.00000") - self.assertEquals(ondisk.normalize_timestamp(1253327593), - "1253327593.00000") - self.assertRaises(ValueError, ondisk.normalize_timestamp, '') - self.assertRaises(ValueError, ondisk.normalize_timestamp, 'abc') - - def test_validate_device_partition(self): - # Test swift.common.ondisk.validate_device_partition - ondisk.validate_device_partition('foo', 'bar') - self.assertRaises(ValueError, - ondisk.validate_device_partition, '', '') - self.assertRaises(ValueError, - ondisk.validate_device_partition, '', 'foo') - self.assertRaises(ValueError, - ondisk.validate_device_partition, 'foo', '') - self.assertRaises(ValueError, - ondisk.validate_device_partition, 'foo/bar', 'foo') - self.assertRaises(ValueError, - ondisk.validate_device_partition, 'foo', 'foo/bar') - self.assertRaises(ValueError, - ondisk.validate_device_partition, '.', 'foo') - self.assertRaises(ValueError, - ondisk.validate_device_partition, '..', 'foo') - self.assertRaises(ValueError, - ondisk.validate_device_partition, 'foo', '.') - self.assertRaises(ValueError, - ondisk.validate_device_partition, 'foo', '..') - try: - ondisk.validate_device_partition('o\nn e', 'foo') - except ValueError as err: - self.assertEquals(str(err), 'Invalid device: o%0An%20e') - try: - ondisk.validate_device_partition('foo', 'o\nn e') - except ValueError as err: - self.assertEquals(str(err), 'Invalid partition: o%0An%20e') - - def test_storage_directory(self): - self.assertEquals(ondisk.storage_directory('objects', '1', 'ABCDEF'), - 'objects/1/DEF/ABCDEF') - - def test_hash_path(self): - _prefix = ondisk.HASH_PATH_PREFIX - ondisk.HASH_PATH_PREFIX = '' - # Yes, these tests are deliberately very fragile. We want to make sure - # that if someones changes the results hash_path produces, they know it - try: - self.assertEquals(ondisk.hash_path('a'), - '1c84525acb02107ea475dcd3d09c2c58') - self.assertEquals(ondisk.hash_path('a', 'c'), - '33379ecb053aa5c9e356c68997cbb59e') - self.assertEquals(ondisk.hash_path('a', 'c', 'o'), - '06fbf0b514e5199dfc4e00f42eb5ea83') - self.assertEquals( - ondisk.hash_path('a', 'c', 'o', raw_digest=False), - '06fbf0b514e5199dfc4e00f42eb5ea83') - self.assertEquals(ondisk.hash_path('a', 'c', 'o', raw_digest=True), - '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN' - '\x00\xf4.\xb5\xea\x83') - self.assertRaises(ValueError, ondisk.hash_path, 'a', object='o') - ondisk.HASH_PATH_PREFIX = 'abcdef' - self.assertEquals( - ondisk.hash_path('a', 'c', 'o', raw_digest=False), - '363f9b535bfb7d17a43a46a358afca0e') - finally: - ondisk.HASH_PATH_PREFIX = _prefix - - -class TestAuditLocationGenerator(unittest.TestCase): - def test_non_dir_contents(self): - with temptree([]) as tmpdir: - data = os.path.join(tmpdir, "drive", "data") - os.makedirs(data) - with open(os.path.join(data, "partition1"), "w"): - pass - partition = os.path.join(data, "partition2") - os.makedirs(partition) - with open(os.path.join(partition, "suffix1"), "w"): - pass - suffix = os.path.join(partition, "suffix2") - os.makedirs(suffix) - with open(os.path.join(suffix, "hash1"), "w"): - pass - locations = ondisk.audit_location_generator( - tmpdir, "data", mount_check=False - ) - self.assertEqual(list(locations), []) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index b4bf59dd80..57d2e561cb 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -133,6 +133,31 @@ def reset_loggers(): class TestUtils(unittest.TestCase): """Tests for swift.common.utils """ + def setUp(self): + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = 'startcap' + + def test_normalize_timestamp(self): + # Test swift.common.utils.normalize_timestamp + self.assertEquals(utils.normalize_timestamp('1253327593.48174'), + "1253327593.48174") + self.assertEquals(utils.normalize_timestamp(1253327593.48174), + "1253327593.48174") + self.assertEquals(utils.normalize_timestamp('1253327593.48'), + "1253327593.48000") + self.assertEquals(utils.normalize_timestamp(1253327593.48), + "1253327593.48000") + self.assertEquals(utils.normalize_timestamp('253327593.48'), + "0253327593.48000") + self.assertEquals(utils.normalize_timestamp(253327593.48), + "0253327593.48000") + self.assertEquals(utils.normalize_timestamp('1253327593'), + "1253327593.00000") + self.assertEquals(utils.normalize_timestamp(1253327593), + "1253327593.00000") + self.assertRaises(ValueError, utils.normalize_timestamp, '') + self.assertRaises(ValueError, utils.normalize_timestamp, 'abc') + def test_backwards(self): # Test swift.common.utils.backward @@ -224,6 +249,36 @@ class TestUtils(unittest.TestCase): except ValueError as err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') + def test_validate_device_partition(self): + # Test swift.common.utils.validate_device_partition + utils.validate_device_partition('foo', 'bar') + self.assertRaises(ValueError, + utils.validate_device_partition, '', '') + self.assertRaises(ValueError, + utils.validate_device_partition, '', 'foo') + self.assertRaises(ValueError, + utils.validate_device_partition, 'foo', '') + self.assertRaises(ValueError, + utils.validate_device_partition, 'foo/bar', 'foo') + self.assertRaises(ValueError, + utils.validate_device_partition, 'foo', 'foo/bar') + self.assertRaises(ValueError, + utils.validate_device_partition, '.', 'foo') + self.assertRaises(ValueError, + utils.validate_device_partition, '..', 'foo') + self.assertRaises(ValueError, + utils.validate_device_partition, 'foo', '.') + self.assertRaises(ValueError, + utils.validate_device_partition, 'foo', '..') + try: + utils.validate_device_partition('o\nn e', 'foo') + except ValueError as err: + self.assertEquals(str(err), 'Invalid device: o%0An%20e') + try: + utils.validate_device_partition('foo', 'o\nn e') + except ValueError as err: + self.assertEquals(str(err), 'Invalid partition: o%0An%20e') + def test_NullLogger(self): # Test swift.common.utils.NullLogger sio = StringIO() @@ -583,6 +638,10 @@ class TestUtils(unittest.TestCase): logger.logger.removeHandler(handler) reset_loggers() + def test_storage_directory(self): + self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), + 'objects/1/DEF/ABCDEF') + def test_whataremyips(self): myips = utils.whataremyips() self.assert_(len(myips) > 1) @@ -618,6 +677,30 @@ class TestUtils(unittest.TestCase): self.assertEquals(len(myips), 1) self.assertEquals(myips[0], test_ipv6_address) + def test_hash_path(self): + _prefix = utils.HASH_PATH_PREFIX + utils.HASH_PATH_PREFIX = '' + # Yes, these tests are deliberately very fragile. We want to make sure + # that if someones changes the results hash_path produces, they know it + try: + self.assertEquals(utils.hash_path('a'), + '1c84525acb02107ea475dcd3d09c2c58') + self.assertEquals(utils.hash_path('a', 'c'), + '33379ecb053aa5c9e356c68997cbb59e') + self.assertEquals(utils.hash_path('a', 'c', 'o'), + '06fbf0b514e5199dfc4e00f42eb5ea83') + self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), + '06fbf0b514e5199dfc4e00f42eb5ea83') + self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True), + '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN' + '\x00\xf4.\xb5\xea\x83') + self.assertRaises(ValueError, utils.hash_path, 'a', object='o') + utils.HASH_PATH_PREFIX = 'abcdef' + self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), + '363f9b535bfb7d17a43a46a358afca0e') + finally: + utils.HASH_PATH_PREFIX = _prefix + def test_load_libc_function(self): self.assert_(callable( utils.load_libc_function('printf'))) @@ -2338,5 +2421,26 @@ class TestThreadpool(unittest.TestCase): self.assertTrue(caught) +class TestAuditLocationGenerator(unittest.TestCase): + def test_non_dir_contents(self): + with temptree([]) as tmpdir: + data = os.path.join(tmpdir, "drive", "data") + os.makedirs(data) + with open(os.path.join(data, "partition1"), "w"): + pass + partition = os.path.join(data, "partition2") + os.makedirs(partition) + with open(os.path.join(partition, "suffix1"), "w"): + pass + suffix = os.path.join(partition, "suffix2") + os.makedirs(suffix) + with open(os.path.join(suffix, "hash1"), "w"): + pass + locations = utils.audit_location_generator( + tmpdir, "data", mount_check=False + ) + self.assertEqual(list(locations), []) + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index cefce984fd..ccc0b250f1 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -22,7 +22,7 @@ from time import sleep, time from uuid import uuid4 from swift.container.backend import ContainerBroker -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp class TestContainerBroker(unittest.TestCase): diff --git a/test/unit/container/test_replicator.py b/test/unit/container/test_replicator.py index 881db43827..8fa92770fb 100644 --- a/test/unit/container/test_replicator.py +++ b/test/unit/container/test_replicator.py @@ -15,7 +15,7 @@ import unittest from swift.container import replicator -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp class TestReplicator(unittest.TestCase): diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index a056af09d3..170aad471b 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -29,8 +29,7 @@ import simplejson from swift.common.swob import Request, HeaderKeyDict import swift.container from swift.container import server as container_server -from swift.common.utils import mkdirs, public, replication -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp, mkdirs, public, replication from test.unit import fake_http_connect diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py index b882b1bfc1..c5a14e36be 100644 --- a/test/unit/container/test_updater.py +++ b/test/unit/container/test_updater.py @@ -28,7 +28,7 @@ from swift.container import updater as container_updater from swift.container import server as container_server from swift.container.backend import ContainerBroker from swift.common.ring import RingData -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import normalize_timestamp class TestContainerUpdater(unittest.TestCase): diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index e173e2109a..23ade8fe8a 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -25,8 +25,7 @@ from test.unit import FakeLogger from swift.obj import auditor from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash from swift.obj.server import DATADIR -from swift.common.utils import mkdirs -from swift.common.ondisk import hash_path, normalize_timestamp, \ +from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ storage_directory diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index bbde9b985a..3962870ec2 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -36,9 +36,8 @@ from eventlet import tpool from test.unit import FakeLogger, mock as unit_mock from test.unit import _setxattr as setxattr from swift.obj import diskfile -from swift.common import ondisk -from swift.common.utils import mkdirs -from swift.common.ondisk import hash_path, normalize_timestamp +from swift.common import utils +from swift.common.utils import hash_path, mkdirs, normalize_timestamp from swift.common import ring from swift.common.exceptions import DiskFileNotExist, DiskFileDeviceUnavailable @@ -73,8 +72,8 @@ def _create_test_ring(path): class TestDiskFileModuleMethods(unittest.TestCase): def setUp(self): - ondisk.HASH_PATH_SUFFIX = 'endcap' - ondisk.HASH_PATH_PREFIX = '' + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = '' # Setup a test ring (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index fafa6dc931..9e7f21991d 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -29,9 +29,8 @@ from eventlet.green import subprocess from eventlet import Timeout, tpool from test.unit import FakeLogger -from swift.common.utils import mkdirs -from swift.common import ondisk -from swift.common.ondisk import hash_path, normalize_timestamp +from swift.common import utils +from swift.common.utils import hash_path, mkdirs, normalize_timestamp from swift.common import ring from swift.obj import diskfile, replicator as object_replicator @@ -138,8 +137,8 @@ def _create_test_ring(path): class TestObjectReplicator(unittest.TestCase): def setUp(self): - ondisk.HASH_PATH_SUFFIX = 'endcap' - ondisk.HASH_PATH_PREFIX = '' + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = '' # Setup a test ring (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 7b0ba51557..b4f2bd35df 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -32,10 +32,9 @@ from test.unit import FakeLogger from test.unit import connect_tcp, readuntil2crlfs from swift.obj import server as object_server from swift.obj import diskfile -from swift.common.utils import mkdirs, NullLogger, public, replication -from swift.common.ondisk import hash_path, normalize_timestamp, \ - storage_directory -from swift.common import ondisk +from swift.common import utils +from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ + NullLogger, storage_directory, public, replication from swift.common import constraints from eventlet import tpool from swift.common.swob import Request, HeaderKeyDict @@ -46,8 +45,8 @@ class TestObjectController(unittest.TestCase): def setUp(self): """Set up for testing swift.object.server.ObjectController""" - ondisk.HASH_PATH_SUFFIX = 'endcap' - ondisk.HASH_PATH_PREFIX = 'startcap' + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = 'startcap' self.testdir = \ os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController') mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) @@ -1905,8 +1904,8 @@ class TestObjectController(unittest.TestCase): 'x-trans-id': '-'})}) def test_async_update_saves_on_exception(self): - _prefix = ondisk.HASH_PATH_PREFIX - ondisk.HASH_PATH_PREFIX = '' + _prefix = utils.HASH_PATH_PREFIX + utils.HASH_PATH_PREFIX = '' def fake_http_connect(*args): raise Exception('test') @@ -1919,7 +1918,7 @@ class TestObjectController(unittest.TestCase): {'x-timestamp': '1', 'x-out': 'set'}, 'sda1') finally: object_server.http_connect = orig_http_connect - ondisk.HASH_PATH_PREFIX = _prefix + utils.HASH_PATH_PREFIX = _prefix self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', 'async_pending', 'a83', @@ -1929,8 +1928,8 @@ class TestObjectController(unittest.TestCase): 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) def test_async_update_saves_on_non_2xx(self): - _prefix = ondisk.HASH_PATH_PREFIX - ondisk.HASH_PATH_PREFIX = '' + _prefix = utils.HASH_PATH_PREFIX + utils.HASH_PATH_PREFIX = '' def fake_http_connect(status): @@ -1964,7 +1963,7 @@ class TestObjectController(unittest.TestCase): 'op': 'PUT'}) finally: object_server.http_connect = orig_http_connect - ondisk.HASH_PATH_PREFIX = _prefix + utils.HASH_PATH_PREFIX = _prefix def test_async_update_does_not_save_on_2xx(self): diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py index d16566de3c..4dbbac0599 100644 --- a/test/unit/obj/test_updater.py +++ b/test/unit/obj/test_updater.py @@ -27,17 +27,17 @@ from eventlet import spawn, Timeout, listen from swift.obj import updater as object_updater, server as object_server from swift.obj.server import ASYNCDIR from swift.common.ring import RingData -from swift.common.utils import mkdirs, write_pickle -from swift.common import ondisk -from swift.common.ondisk import hash_path, normalize_timestamp +from swift.common import utils +from swift.common.utils import hash_path, normalize_timestamp, mkdirs, \ + write_pickle from test.unit import FakeLogger class TestObjectUpdater(unittest.TestCase): def setUp(self): - ondisk.HASH_PATH_SUFFIX = 'endcap' - ondisk.HASH_PATH_PREFIX = '' + utils.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'object_updater') rmtree(self.testdir, ignore_errors=1) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 3ad5c7ae2e..47adaadd01 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -47,9 +47,7 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH from swift.common import utils -from swift.common.utils import mkdirs, NullLogger -from swift.common import ondisk -from swift.common.ondisk import normalize_timestamp +from swift.common.utils import mkdirs, normalize_timestamp, NullLogger from swift.common.wsgi import monkey_patch_mimetools from swift.proxy.controllers.obj import SegmentedIterable from swift.proxy.controllers.base import get_container_memcache_key, \ @@ -75,7 +73,7 @@ def request_init(self, *args, **kwargs): def setup(): - ondisk.HASH_PATH_SUFFIX = 'endcap' + utils.HASH_PATH_SUFFIX = 'endcap' global _testdir, _test_servers, _test_sockets, \ _orig_container_listing_limit, _test_coros, _orig_SysLogHandler _orig_SysLogHandler = utils.SysLogHandler