diff --git a/doc/source/misc.rst b/doc/source/misc.rst index e42a91193d..e8921173f1 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -33,6 +33,15 @@ Utils :members: :show-inheritance: +.. _ondisk: + +OnDisk +====== + +.. automodule:: swift.common.ondisk + :members: + :show-inheritance: + .. _common_tempauth: TempAuth diff --git a/swift/account/auditor.py b/swift/account/auditor.py index 5f81490d94..a85839ad70 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -21,8 +21,9 @@ from random import random import swift.common.db from swift.account import server as account_server from swift.account.backend import AccountBroker -from swift.common.utils import get_logger, audit_location_generator, \ - config_true_value, dump_recon_cache, ratelimit_sleep +from swift.common.utils import get_logger, config_true_value, \ + dump_recon_cache, ratelimit_sleep +from swift.common.ondisk import audit_location_generator from swift.common.daemon import Daemon from eventlet import Timeout diff --git a/swift/account/backend.py b/swift/account/backend.py index a772ba2a5f..6a889fbb7f 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -25,7 +25,8 @@ import errno import sqlite3 -from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.utils import lock_parent_directory +from swift.common.ondisk import normalize_timestamp from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode diff --git a/swift/account/server.py b/swift/account/server.py index 99bfc58cd6..da27fe613f 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -28,9 +28,10 @@ from swift.account.utils import account_listing_response from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path -from swift.common.utils import get_logger, hash_path, public, \ - normalize_timestamp, storage_directory, config_true_value, \ +from swift.common.utils import get_logger, public, config_true_value, \ json, timing_stats, replication +from swift.common.ondisk import hash_path, normalize_timestamp, \ + storage_directory from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc diff --git a/swift/account/utils.py b/swift/account/utils.py index 5565b46166..919c05c02d 100644 --- a/swift/account/utils.py +++ b/swift/account/utils.py @@ -17,7 +17,8 @@ import time from xml.sax import saxutils from swift.common.swob import HTTPOk, HTTPNoContent -from swift.common.utils import json, normalize_timestamp +from swift.common.utils import json +from swift.common.ondisk import normalize_timestamp class FakeAccountBroker(object): diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 0b3effb542..25de076c66 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -22,6 +22,7 @@ from re import sub import eventlet.debug from swift.common import utils +from swift.common import ondisk class Daemon(object): @@ -41,7 +42,7 @@ class Daemon(object): def run(self, once=False, **kwargs): """Run the daemon""" - utils.validate_configuration() + ondisk.validate_configuration() utils.drop_privileges(self.conf.get('user', 'swift')) utils.capture_stdio(self.logger, **kwargs) diff --git a/swift/common/db.py b/swift/common/db.py index 20fec973af..0fd4900e01 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -30,8 +30,9 @@ from tempfile import mkstemp from eventlet import sleep, Timeout import sqlite3 -from swift.common.utils import json, normalize_timestamp, renamer, \ - mkdirs, lock_parent_directory, fallocate +from swift.common.utils import json, renamer, mkdirs, lock_parent_directory, \ + fallocate +from swift.common.ondisk import normalize_timestamp from swift.common.exceptions import LockTimeout diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 32fe485c8d..b0965296f2 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -30,9 +30,10 @@ import simplejson import swift.common.db from swift.common.direct_client import quote -from swift.common.utils import get_logger, whataremyips, storage_directory, \ - renamer, mkdirs, lock_parent_directory, config_true_value, \ - unlink_older_than, dump_recon_cache, rsync_ip +from swift.common.utils import get_logger, whataremyips, renamer, mkdirs, \ + lock_parent_directory, config_true_value, unlink_older_than, \ + dump_recon_cache, rsync_ip +from swift.common.ondisk import storage_directory from swift.common import ring from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE from swift.common.bufferedhttp import BufferedHTTPConnection diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 36e495ca79..ab283026d1 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -27,7 +27,8 @@ from eventlet import sleep, Timeout from swift.common.bufferedhttp import http_connect from swiftclient import ClientException, json_loads -from swift.common.utils import normalize_timestamp, FileLikeIter +from swift.common.utils import FileLikeIter +from swift.common.ondisk import normalize_timestamp from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \ is_success, is_server_error from swift.common.swob import HeaderKeyDict diff --git a/swift/common/ondisk.py b/swift/common/ondisk.py new file mode 100644 index 0000000000..8acad105c2 --- /dev/null +++ b/swift/common/ondisk.py @@ -0,0 +1,184 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Methods & Attributes for shared 'on-disk' data layouts.""" + +import os +import sys +import errno + +from hashlib import md5 +from random import shuffle +from ConfigParser import ConfigParser, NoSectionError, NoOptionError + +from swift import gettext_ as _ +from swift.common.utils import listdir, quote + +# Used by hash_path to offer a bit more security when generating hashes for +# paths. It simply appends this value to all paths; guessing the hash a path +# will end up with would also require knowing this suffix. +_hash_conf = ConfigParser() +HASH_PATH_SUFFIX = '' +HASH_PATH_PREFIX = '' +if _hash_conf.read('/etc/swift/swift.conf'): + try: + HASH_PATH_SUFFIX = _hash_conf.get('swift-hash', + 'swift_hash_path_suffix') + except (NoSectionError, NoOptionError): + pass + try: + HASH_PATH_PREFIX = _hash_conf.get('swift-hash', + 'swift_hash_path_prefix') + except (NoSectionError, NoOptionError): + pass + + +def validate_configuration(): + if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX: + sys.exit("Error: [swift-hash]: both swift_hash_path_suffix " + "and swift_hash_path_prefix are missing " + "from /etc/swift/swift.conf") + + +def hash_path(account, container=None, object=None, raw_digest=False): + """ + Get the canonical hash for an account/container/object + + :param account: Account + :param container: Container + :param object: Object + :param raw_digest: If True, return the raw version rather than a hex digest + :returns: hash string + """ + if object and not container: + raise ValueError('container is required if object is provided') + paths = [account] + if container: + paths.append(container) + if object: + paths.append(object) + if raw_digest: + return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) + + HASH_PATH_SUFFIX).digest() + else: + return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) + + HASH_PATH_SUFFIX).hexdigest() + + +def normalize_timestamp(timestamp): + """ + Format a timestamp (string or numeric) into a standardized + xxxxxxxxxx.xxxxx (10.5) format. + + Note that timestamps using values greater than or equal to November 20th, + 2286 at 17:46 UTC will use 11 digits to represent the number of + seconds. + + :param timestamp: unix timestamp + :returns: normalized timestamp as a string + """ + return "%016.05f" % (float(timestamp)) + + +def validate_device_partition(device, partition): + """ + Validate that a device and a partition are valid and won't lead to + directory traversal when used. + + :param device: device to validate + :param partition: partition to validate + :raises: ValueError if given an invalid device or partition + """ + invalid_device = False + invalid_partition = False + if not device or '/' in device or device in ['.', '..']: + invalid_device = True + if not partition or '/' in partition or partition in ['.', '..']: + invalid_partition = True + + if invalid_device: + raise ValueError('Invalid device: %s' % quote(device or '')) + elif invalid_partition: + raise ValueError('Invalid partition: %s' % quote(partition or '')) + + +def storage_directory(datadir, partition, name_hash): + """ + Get the storage directory + + :param datadir: Base data directory + :param partition: Partition + :param name_hash: Account, container or object name hash + :returns: Storage directory + """ + return os.path.join(datadir, str(partition), name_hash[-3:], name_hash) + + +def audit_location_generator(devices, datadir, suffix='', + mount_check=True, logger=None): + ''' + Given a devices path and a data directory, yield (path, device, + partition) for all files in that directory + + :param devices: parent directory of the devices to be audited + :param datadir: a directory located under self.devices. This should be + one of the DATADIR constants defined in the account, + container, and object servers. + :param suffix: path name suffix required for all names returned + :param mount_check: Flag to check if a mount check should be performed + on devices + :param logger: a logger object + ''' + device_dir = listdir(devices) + # randomize devices in case of process restart before sweep completed + shuffle(device_dir) + for device in device_dir: + if mount_check and not \ + os.path.ismount(os.path.join(devices, device)): + if logger: + logger.debug( + _('Skipping %s as it is not mounted'), device) + continue + datadir_path = os.path.join(devices, device, datadir) + partitions = listdir(datadir_path) + for partition in partitions: + part_path = os.path.join(datadir_path, partition) + try: + suffixes = listdir(part_path) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for asuffix in suffixes: + suff_path = os.path.join(part_path, asuffix) + try: + hashes = listdir(suff_path) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for hsh in hashes: + hash_path = os.path.join(suff_path, hsh) + try: + files = sorted(listdir(hash_path), reverse=True) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for fname in files: + if suffix and not fname.endswith(suffix): + continue + path = os.path.join(hash_path, fname) + yield path, device, partition diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index b24c8b24c8..51c846d124 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -22,7 +22,8 @@ from swob in here without creating circular imports. from swift.common.constraints import FORMAT2CONTENT_TYPE from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable -from swift.common.utils import split_path, validate_device_partition +from swift.common.ondisk import validate_device_partition +from swift.common.utils import split_path from urllib import unquote diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 9ed0b35bfd..22bdb4c442 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -25,7 +25,8 @@ from io import BufferedReader from hashlib import md5 from itertools import chain -from swift.common.utils import hash_path, validate_configuration, json +from swift.common.utils import json +from swift.common.ondisk import hash_path, validate_configuration from swift.common.ring.utils import tiers_for_dev @@ -130,7 +131,7 @@ class Ring(object): """ def __init__(self, serialized_path, reload_time=15, ring_name=None): - # can't use the ring unless HASH_PATH_SUFFIX is set + # Can't use the ring unless the on-disk configuration is valid validate_configuration() if ring_name: self.serialized_path = os.path.join(serialized_path, diff --git a/swift/common/utils.py b/swift/common/utils.py index 08cf0b04ad..8173cbbdd2 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -26,14 +26,12 @@ import threading as stdlib_threading import time import uuid import functools -from hashlib import md5 -from random import random, shuffle +from random import random from urllib import quote as _quote from contextlib import contextmanager, closing import ctypes import ctypes.util -from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \ - RawConfigParser +from ConfigParser import ConfigParser, RawConfigParser from optparse import OptionParser from Queue import Queue, Empty from tempfile import mkstemp, NamedTemporaryFile @@ -80,24 +78,6 @@ _posix_fadvise = None # available being at or below this amount, in bytes. FALLOCATE_RESERVE = 0 -# Used by hash_path to offer a bit more security when generating hashes for -# paths. It simply appends this value to all paths; guessing the hash a path -# will end up with would also require knowing this suffix. -hash_conf = ConfigParser() -HASH_PATH_SUFFIX = '' -HASH_PATH_PREFIX = '' -if hash_conf.read('/etc/swift/swift.conf'): - try: - HASH_PATH_SUFFIX = hash_conf.get('swift-hash', - 'swift_hash_path_suffix') - except (NoSectionError, NoOptionError): - pass - try: - HASH_PATH_PREFIX = hash_conf.get('swift-hash', - 'swift_hash_path_prefix') - except (NoSectionError, NoOptionError): - pass - def backward(f, blocksize=4096): """ @@ -164,13 +144,6 @@ def noop_libc_function(*args): return 0 -def validate_configuration(): - if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX: - sys.exit("Error: [swift-hash]: both swift_hash_path_suffix " - "and swift_hash_path_prefix are missing " - "from /etc/swift/swift.conf") - - def load_libc_function(func_name, log_error=True): """ Attempt to find the function in libc, otherwise return a no-op func. @@ -422,21 +395,6 @@ def drop_buffer_cache(fd, offset, length): % (fd, offset, length, ret)) -def normalize_timestamp(timestamp): - """ - Format a timestamp (string or numeric) into a standardized - xxxxxxxxxx.xxxxx (10.5) format. - - Note that timestamps using values greater than or equal to November 20th, - 2286 at 17:46 UTC will use 11 digits to represent the number of - seconds. - - :param timestamp: unix timestamp - :returns: normalized timestamp as a string - """ - return "%016.05f" % (float(timestamp)) - - def mkdirs(path): """ Ensures the path is a directory or makes it if not. Errors if the path @@ -515,28 +473,6 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): return segs -def validate_device_partition(device, partition): - """ - Validate that a device and a partition are valid and won't lead to - directory traversal when used. - - :param device: device to validate - :param partition: partition to validate - :raises: ValueError if given an invalid device or partition - """ - invalid_device = False - invalid_partition = False - if not device or '/' in device or device in ['.', '..']: - invalid_device = True - if not partition or '/' in partition or partition in ['.', '..']: - invalid_partition = True - - if invalid_device: - raise ValueError('Invalid device: %s' % quote(device or '')) - elif invalid_partition: - raise ValueError('Invalid partition: %s' % quote(partition or '')) - - class GreenthreadSafeIterator(object): """ Wrap an iterator to ensure that only one greenthread is inside its next() @@ -1140,43 +1076,6 @@ def whataremyips(): return addresses -def storage_directory(datadir, partition, name_hash): - """ - Get the storage directory - - :param datadir: Base data directory - :param partition: Partition - :param name_hash: Account, container or object name hash - :returns: Storage directory - """ - return os.path.join(datadir, str(partition), name_hash[-3:], name_hash) - - -def hash_path(account, container=None, object=None, raw_digest=False): - """ - Get the canonical hash for an account/container/object - - :param account: Account - :param container: Container - :param object: Object - :param raw_digest: If True, return the raw version rather than a hex digest - :returns: hash string - """ - if object and not container: - raise ValueError('container is required if object is provided') - paths = [account] - if container: - paths.append(container) - if object: - paths.append(object) - if raw_digest: - return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) - + HASH_PATH_SUFFIX).digest() - else: - return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) - + HASH_PATH_SUFFIX).hexdigest() - - @contextmanager def lock_path(directory, timeout=10): """ @@ -1491,64 +1390,6 @@ def remove_file(path): pass -def audit_location_generator(devices, datadir, suffix='', - mount_check=True, logger=None): - ''' - Given a devices path and a data directory, yield (path, device, - partition) for all files in that directory - - :param devices: parent directory of the devices to be audited - :param datadir: a directory located under self.devices. This should be - one of the DATADIR constants defined in the account, - container, and object servers. - :param suffix: path name suffix required for all names returned - :param mount_check: Flag to check if a mount check should be performed - on devices - :param logger: a logger object - ''' - device_dir = listdir(devices) - # randomize devices in case of process restart before sweep completed - shuffle(device_dir) - for device in device_dir: - if mount_check and not \ - os.path.ismount(os.path.join(devices, device)): - if logger: - logger.debug( - _('Skipping %s as it is not mounted'), device) - continue - datadir_path = os.path.join(devices, device, datadir) - partitions = listdir(datadir_path) - for partition in partitions: - part_path = os.path.join(datadir_path, partition) - try: - suffixes = listdir(part_path) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for asuffix in suffixes: - suff_path = os.path.join(part_path, asuffix) - try: - hashes = listdir(suff_path) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for hsh in hashes: - hash_path = os.path.join(suff_path, hsh) - try: - files = sorted(listdir(hash_path), reverse=True) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for fname in files: - if suffix and not fname.endswith(suffix): - continue - path = os.path.join(hash_path, fname) - yield path, device, partition - - def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5): ''' Will eventlet.sleep() for the appropriate time so that the max_rate diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index d8f7d3bcec..f8ff8c2e82 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -35,7 +35,8 @@ from swift.common import utils from swift.common.swob import Request from swift.common.utils import capture_stdio, disable_fallocate, \ drop_privileges, get_logger, NullLogger, config_true_value, \ - validate_configuration, get_hub, config_auto_int_value + get_hub, config_auto_int_value +from swift.common.ondisk import validate_configuration try: import multiprocessing diff --git a/swift/container/auditor.py b/swift/container/auditor.py index df2266c076..bfcefbc53a 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -23,8 +23,9 @@ from eventlet import Timeout import swift.common.db from swift.container import server as container_server from swift.container.backend import ContainerBroker -from swift.common.utils import get_logger, audit_location_generator, \ - config_true_value, dump_recon_cache, ratelimit_sleep +from swift.common.utils import get_logger, config_true_value, \ + dump_recon_cache, ratelimit_sleep +from swift.common.ondisk import audit_location_generator from swift.common.daemon import Daemon diff --git a/swift/container/backend.py b/swift/container/backend.py index 1723c9c93c..f81b946b89 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -25,7 +25,8 @@ import errno import sqlite3 -from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.utils import lock_parent_directory +from swift.common.ondisk import normalize_timestamp from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode diff --git a/swift/container/server.py b/swift/container/server.py index 9013ce8c57..2e08bba890 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -29,9 +29,10 @@ from swift.container.backend import ContainerBroker from swift.common.db import DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path -from swift.common.utils import get_logger, hash_path, public, \ - normalize_timestamp, storage_directory, validate_sync_to, \ +from swift.common.utils import get_logger, public, validate_sync_to, \ config_true_value, json, timing_stats, replication, parse_content_type +from swift.common.ondisk import hash_path, normalize_timestamp, \ + storage_directory from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.bufferedhttp import http_connect diff --git a/swift/container/sync.py b/swift/container/sync.py index 759248417b..c014914b59 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -27,8 +27,9 @@ from swiftclient import ClientException, delete_object, put_object, \ from swift.container.backend import ContainerBroker from swift.common.direct_client import direct_get_object from swift.common.ring import Ring -from swift.common.utils import audit_location_generator, get_logger, \ - hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter +from swift.common.utils import get_logger, config_true_value, \ + validate_sync_to, whataremyips, FileLikeIter +from swift.common.ondisk import audit_location_generator, hash_path from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 78ab123474..cd674b7e79 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -21,8 +21,9 @@ from eventlet import Timeout from swift.obj import diskfile from swift.obj import server as object_server -from swift.common.utils import get_logger, audit_location_generator, \ - ratelimit_sleep, config_true_value, dump_recon_cache, list_from_csv, json +from swift.common.utils import get_logger, ratelimit_sleep, \ + config_true_value, dump_recon_cache, list_from_csv, json +from swift.common.ondisk import audit_location_generator from swift.common.exceptions import AuditException, DiskFileError, \ DiskFileNotExist from swift.common.daemon import Daemon diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index cec98c728a..b8326d3e45 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -33,9 +33,10 @@ from eventlet import Timeout from swift import gettext_ as _ from swift.common.constraints import check_mount -from swift.common.utils import mkdirs, normalize_timestamp, \ - storage_directory, hash_path, renamer, fallocate, fsync, \ - fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle +from swift.common.utils import mkdirs, renamer, fallocate, fsync, fdatasync, \ + drop_buffer_cache, ThreadPool, lock_path, write_pickle +from swift.common.ondisk import hash_path, normalize_timestamp, \ + storage_directory from swift.common.exceptions import DiskFileError, DiskFileNotExist, \ DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \ PathNotDir, DiskFileNotOpenError diff --git a/swift/obj/server.py b/swift/obj/server.py index 362f940197..a89c8ff2b7 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -27,9 +27,9 @@ from hashlib import md5 from eventlet import sleep, Timeout -from swift.common.utils import mkdirs, normalize_timestamp, public, \ - hash_path, get_logger, write_pickle, config_true_value, timing_stats, \ - ThreadPool, replication +from swift.common.utils import mkdirs, public, get_logger, write_pickle, \ + config_true_value, timing_stats, ThreadPool, replication +from swift.common.ondisk import normalize_timestamp, hash_path from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 1e0b7146da..f1c1640cd4 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -37,9 +37,9 @@ from eventlet.queue import Queue, Empty, Full from eventlet.timeout import Timeout from swift.common.wsgi import make_pre_authed_env -from swift.common.utils import normalize_timestamp, config_true_value, \ - public, split_path, list_from_csv, GreenthreadSafeIterator, \ - quorum_size +from swift.common.utils import config_true_value, public, split_path, \ + list_from_csv, GreenthreadSafeIterator, quorum_size +from swift.common.ondisk import normalize_timestamp from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout from swift.common.http import is_informational, is_success, is_redirection, \ diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 09e4016e46..ee6afdac1c 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -37,9 +37,9 @@ from eventlet import sleep, GreenPile from eventlet.queue import Queue from eventlet.timeout import Timeout -from swift.common.utils import ContextPool, normalize_timestamp, \ - config_true_value, public, json, csv_append, GreenthreadSafeIterator, \ - quorum_size +from swift.common.utils import ContextPool, config_true_value, public, json, \ + csv_append, GreenthreadSafeIterator, quorum_size +from swift.common.ondisk import normalize_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE, MAX_BUFFERED_SLO_SEGMENTS diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py index 379598ba5f..2f59f9a4a6 100644 --- a/test/unit/account/test_backend.py +++ b/test/unit/account/test_backend.py @@ -22,7 +22,7 @@ from time import sleep, time from uuid import uuid4 from swift.account.backend import AccountBroker -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp class TestAccountBroker(unittest.TestCase): diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index e0f438b80c..81ea63b5e2 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -25,7 +25,7 @@ from contextlib import nested from swift.account import reaper from swift.account.server import DATADIR -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp from swift.common.direct_client import ClientException diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 8461071409..ba56b6d85b 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -25,7 +25,8 @@ import xml.dom.minidom from swift.common.swob import Request from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT -from swift.common.utils import normalize_timestamp, replication, public +from swift.common.utils import replication, public +from swift.common.ondisk import normalize_timestamp class TestAccountController(unittest.TestCase): diff --git a/test/unit/common/middleware/test_list_endpoints.py b/test/unit/common/middleware/test_list_endpoints.py index 6d9f6d3d44..8da02382ae 100644 --- a/test/unit/common/middleware/test_list_endpoints.py +++ b/test/unit/common/middleware/test_list_endpoints.py @@ -18,7 +18,7 @@ import unittest from shutil import rmtree import os -from swift.common import ring, utils +from swift.common import ring, ondisk from swift.common.utils import json from swift.common.swob import Request, Response from swift.common.middleware import list_endpoints @@ -35,8 +35,8 @@ def start_response(*args): class TestListEndpoints(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'ring') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index 3d5d8f2b1c..d47ee564f0 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -23,7 +23,7 @@ from gzip import GzipFile from shutil import rmtree from time import sleep, time -from swift.common import ring, utils +from swift.common import ring, ondisk class TestRingData(unittest.TestCase): @@ -99,8 +99,8 @@ class TestRingData(unittest.TestCase): class TestRing(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'ring') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) @@ -146,15 +146,15 @@ class TestRing(unittest.TestCase): self.assertEquals(self.ring.reload_time, self.intended_reload_time) self.assertEquals(self.ring.serialized_path, self.testgz) # test invalid endcap - _orig_hash_path_suffix = utils.HASH_PATH_SUFFIX - _orig_hash_path_prefix = utils.HASH_PATH_PREFIX + _orig_hash_path_suffix = ondisk.HASH_PATH_SUFFIX + _orig_hash_path_prefix = ondisk.HASH_PATH_PREFIX try: - utils.HASH_PATH_SUFFIX = '' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = '' + ondisk.HASH_PATH_PREFIX = '' self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever') finally: - utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix - utils.HASH_PATH_PREFIX = _orig_hash_path_prefix + ondisk.HASH_PATH_SUFFIX = _orig_hash_path_suffix + ondisk.HASH_PATH_PREFIX = _orig_hash_path_prefix def test_has_changed(self): self.assertEquals(self.ring.has_changed(), False) diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index a0eb6caecb..246610bdf4 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -23,7 +23,7 @@ from StringIO import StringIO from test.unit import tmpfile from mock import patch -from swift.common import daemon, utils +from swift.common import daemon, utils, ondisk class MyDaemon(daemon.Daemon): @@ -63,8 +63,8 @@ class TestDaemon(unittest.TestCase): class TestRunDaemon(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = 'startcap' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = 'startcap' utils.drop_privileges = lambda *args: None utils.capture_stdio = lambda *args: None diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 5bd3fb9c1c..1ae67df3c1 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -28,7 +28,7 @@ from mock import patch import swift.common.db from swift.common.db import chexor, dict_factory, get_db_connection, \ DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp from swift.common.exceptions import LockTimeout diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index 04173f5585..6913b691eb 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -26,7 +26,7 @@ import mock import simplejson from swift.common import db_replicator -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp from swift.container import server as container_server from swift.common.exceptions import DriveNotMounted diff --git a/test/unit/common/test_ondisk.py b/test/unit/common/test_ondisk.py new file mode 100644 index 0000000000..394d008ed0 --- /dev/null +++ b/test/unit/common/test_ondisk.py @@ -0,0 +1,139 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for swift.common.ondisk""" + +from __future__ import with_statement +from test.unit import temptree + +import os + +import unittest + +from swift.common import ondisk + + +class TestOndisk(unittest.TestCase): + """Tests for swift.common.ondisk""" + + def setUp(self): + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = 'startcap' + + def test_normalize_timestamp(self): + # Test swift.common.ondisk.normalize_timestamp + self.assertEquals(ondisk.normalize_timestamp('1253327593.48174'), + "1253327593.48174") + self.assertEquals(ondisk.normalize_timestamp(1253327593.48174), + "1253327593.48174") + self.assertEquals(ondisk.normalize_timestamp('1253327593.48'), + "1253327593.48000") + self.assertEquals(ondisk.normalize_timestamp(1253327593.48), + "1253327593.48000") + self.assertEquals(ondisk.normalize_timestamp('253327593.48'), + "0253327593.48000") + self.assertEquals(ondisk.normalize_timestamp(253327593.48), + "0253327593.48000") + self.assertEquals(ondisk.normalize_timestamp('1253327593'), + "1253327593.00000") + self.assertEquals(ondisk.normalize_timestamp(1253327593), + "1253327593.00000") + self.assertRaises(ValueError, ondisk.normalize_timestamp, '') + self.assertRaises(ValueError, ondisk.normalize_timestamp, 'abc') + + def test_validate_device_partition(self): + # Test swift.common.ondisk.validate_device_partition + ondisk.validate_device_partition('foo', 'bar') + self.assertRaises(ValueError, + ondisk.validate_device_partition, '', '') + self.assertRaises(ValueError, + ondisk.validate_device_partition, '', 'foo') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo', '') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo/bar', 'foo') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo', 'foo/bar') + self.assertRaises(ValueError, + ondisk.validate_device_partition, '.', 'foo') + self.assertRaises(ValueError, + ondisk.validate_device_partition, '..', 'foo') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo', '.') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo', '..') + try: + ondisk.validate_device_partition('o\nn e', 'foo') + except ValueError as err: + self.assertEquals(str(err), 'Invalid device: o%0An%20e') + try: + ondisk.validate_device_partition('foo', 'o\nn e') + except ValueError as err: + self.assertEquals(str(err), 'Invalid partition: o%0An%20e') + + def test_storage_directory(self): + self.assertEquals(ondisk.storage_directory('objects', '1', 'ABCDEF'), + 'objects/1/DEF/ABCDEF') + + def test_hash_path(self): + _prefix = ondisk.HASH_PATH_PREFIX + ondisk.HASH_PATH_PREFIX = '' + # Yes, these tests are deliberately very fragile. We want to make sure + # that if someones changes the results hash_path produces, they know it + try: + self.assertEquals(ondisk.hash_path('a'), + '1c84525acb02107ea475dcd3d09c2c58') + self.assertEquals(ondisk.hash_path('a', 'c'), + '33379ecb053aa5c9e356c68997cbb59e') + self.assertEquals(ondisk.hash_path('a', 'c', 'o'), + '06fbf0b514e5199dfc4e00f42eb5ea83') + self.assertEquals( + ondisk.hash_path('a', 'c', 'o', raw_digest=False), + '06fbf0b514e5199dfc4e00f42eb5ea83') + self.assertEquals(ondisk.hash_path('a', 'c', 'o', raw_digest=True), + '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN' + '\x00\xf4.\xb5\xea\x83') + self.assertRaises(ValueError, ondisk.hash_path, 'a', object='o') + ondisk.HASH_PATH_PREFIX = 'abcdef' + self.assertEquals( + ondisk.hash_path('a', 'c', 'o', raw_digest=False), + '363f9b535bfb7d17a43a46a358afca0e') + finally: + ondisk.HASH_PATH_PREFIX = _prefix + + +class TestAuditLocationGenerator(unittest.TestCase): + def test_non_dir_contents(self): + with temptree([]) as tmpdir: + data = os.path.join(tmpdir, "drive", "data") + os.makedirs(data) + with open(os.path.join(data, "partition1"), "w"): + pass + partition = os.path.join(data, "partition2") + os.makedirs(partition) + with open(os.path.join(partition, "suffix1"), "w"): + pass + suffix = os.path.join(partition, "suffix2") + os.makedirs(suffix) + with open(os.path.join(suffix, "hash1"), "w"): + pass + locations = ondisk.audit_location_generator( + tmpdir, "data", mount_check=False + ) + self.assertEqual(list(locations), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index cf1d1a9155..c9408eab52 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -132,31 +132,6 @@ def reset_loggers(): class TestUtils(unittest.TestCase): """Tests for swift.common.utils """ - def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = 'startcap' - - def test_normalize_timestamp(self): - # Test swift.common.utils.normalize_timestamp - self.assertEquals(utils.normalize_timestamp('1253327593.48174'), - "1253327593.48174") - self.assertEquals(utils.normalize_timestamp(1253327593.48174), - "1253327593.48174") - self.assertEquals(utils.normalize_timestamp('1253327593.48'), - "1253327593.48000") - self.assertEquals(utils.normalize_timestamp(1253327593.48), - "1253327593.48000") - self.assertEquals(utils.normalize_timestamp('253327593.48'), - "0253327593.48000") - self.assertEquals(utils.normalize_timestamp(253327593.48), - "0253327593.48000") - self.assertEquals(utils.normalize_timestamp('1253327593'), - "1253327593.00000") - self.assertEquals(utils.normalize_timestamp(1253327593), - "1253327593.00000") - self.assertRaises(ValueError, utils.normalize_timestamp, '') - self.assertRaises(ValueError, utils.normalize_timestamp, 'abc') - def test_backwards(self): # Test swift.common.utils.backward @@ -248,36 +223,6 @@ class TestUtils(unittest.TestCase): except ValueError as err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') - def test_validate_device_partition(self): - # Test swift.common.utils.validate_device_partition - utils.validate_device_partition('foo', 'bar') - self.assertRaises(ValueError, - utils.validate_device_partition, '', '') - self.assertRaises(ValueError, - utils.validate_device_partition, '', 'foo') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo', '') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo/bar', 'foo') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo', 'foo/bar') - self.assertRaises(ValueError, - utils.validate_device_partition, '.', 'foo') - self.assertRaises(ValueError, - utils.validate_device_partition, '..', 'foo') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo', '.') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo', '..') - try: - utils.validate_device_partition('o\nn e', 'foo') - except ValueError as err: - self.assertEquals(str(err), 'Invalid device: o%0An%20e') - try: - utils.validate_device_partition('foo', 'o\nn e') - except ValueError as err: - self.assertEquals(str(err), 'Invalid partition: o%0An%20e') - def test_NullLogger(self): # Test swift.common.utils.NullLogger sio = StringIO() @@ -637,10 +582,6 @@ class TestUtils(unittest.TestCase): logger.logger.removeHandler(handler) reset_loggers() - def test_storage_directory(self): - self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), - 'objects/1/DEF/ABCDEF') - def test_whataremyips(self): myips = utils.whataremyips() self.assert_(len(myips) > 1) @@ -676,30 +617,6 @@ class TestUtils(unittest.TestCase): self.assertEquals(len(myips), 1) self.assertEquals(myips[0], test_ipv6_address) - def test_hash_path(self): - _prefix = utils.HASH_PATH_PREFIX - utils.HASH_PATH_PREFIX = '' - # Yes, these tests are deliberately very fragile. We want to make sure - # that if someones changes the results hash_path produces, they know it - try: - self.assertEquals(utils.hash_path('a'), - '1c84525acb02107ea475dcd3d09c2c58') - self.assertEquals(utils.hash_path('a', 'c'), - '33379ecb053aa5c9e356c68997cbb59e') - self.assertEquals(utils.hash_path('a', 'c', 'o'), - '06fbf0b514e5199dfc4e00f42eb5ea83') - self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), - '06fbf0b514e5199dfc4e00f42eb5ea83') - self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True), - '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN' - '\x00\xf4.\xb5\xea\x83') - self.assertRaises(ValueError, utils.hash_path, 'a', object='o') - utils.HASH_PATH_PREFIX = 'abcdef' - self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), - '363f9b535bfb7d17a43a46a358afca0e') - finally: - utils.HASH_PATH_PREFIX = _prefix - def test_load_libc_function(self): self.assert_(callable( utils.load_libc_function('printf'))) @@ -2401,26 +2318,5 @@ class TestThreadpool(unittest.TestCase): self.assertTrue(caught) -class TestAuditLocationGenerator(unittest.TestCase): - def test_non_dir_contents(self): - with temptree([]) as tmpdir: - data = os.path.join(tmpdir, "drive", "data") - os.makedirs(data) - with open(os.path.join(data, "partition1"), "w"): - pass - partition = os.path.join(data, "partition2") - os.makedirs(partition) - with open(os.path.join(partition, "suffix1"), "w"): - pass - suffix = os.path.join(partition, "suffix2") - os.makedirs(suffix) - with open(os.path.join(suffix, "hash1"), "w"): - pass - locations = utils.audit_location_generator( - tmpdir, "data", mount_check=False - ) - self.assertEqual(list(locations), []) - - if __name__ == '__main__': unittest.main() diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index 3bda3ccef3..ca7a841fa9 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -22,7 +22,7 @@ from time import sleep, time from uuid import uuid4 from swift.container.backend import ContainerBroker -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp class TestContainerBroker(unittest.TestCase): diff --git a/test/unit/container/test_replicator.py b/test/unit/container/test_replicator.py index 9f1b0259a8..cb31a0e5f0 100644 --- a/test/unit/container/test_replicator.py +++ b/test/unit/container/test_replicator.py @@ -15,7 +15,7 @@ import unittest from swift.container import replicator -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp class TestReplicator(unittest.TestCase): diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index dddff8f6f8..627d67cdac 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -29,7 +29,8 @@ import simplejson from swift.common.swob import Request, HeaderKeyDict import swift.container from swift.container import server as container_server -from swift.common.utils import normalize_timestamp, mkdirs, public, replication +from swift.common.utils import mkdirs, public, replication +from swift.common.ondisk import normalize_timestamp from test.unit import fake_http_connect diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py index 6b6030bd86..68bc9dd695 100644 --- a/test/unit/container/test_updater.py +++ b/test/unit/container/test_updater.py @@ -28,7 +28,7 @@ from swift.container import updater as container_updater from swift.container import server as container_server from swift.container.backend import ContainerBroker from swift.common.ring import RingData -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp class TestContainerUpdater(unittest.TestCase): diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index bbd7070ba8..deca216eae 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -25,7 +25,8 @@ from test.unit import FakeLogger from swift.obj import auditor from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash from swift.obj.server import DATADIR -from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ +from swift.common.utils import mkdirs +from swift.common.ondisk import hash_path, normalize_timestamp, \ storage_directory diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 7d0cbb58ea..c5db32a668 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -36,8 +36,9 @@ from eventlet import tpool from test.unit import FakeLogger, mock as unit_mock from test.unit import _setxattr as setxattr from swift.obj import diskfile -from swift.common import utils -from swift.common.utils import hash_path, mkdirs, normalize_timestamp +from swift.common import ondisk +from swift.common.utils import mkdirs +from swift.common.ondisk import hash_path, normalize_timestamp from swift.common import ring from swift.common.exceptions import DiskFileNotExist, DiskFileDeviceUnavailable @@ -72,8 +73,8 @@ def _create_test_ring(path): class TestDiskFileModuleMethods(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' # Setup a test ring (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index c8805494f5..ac79f717cb 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -29,8 +29,9 @@ from eventlet.green import subprocess from eventlet import Timeout, tpool from test.unit import FakeLogger -from swift.common import utils -from swift.common.utils import hash_path, mkdirs, normalize_timestamp +from swift.common.utils import mkdirs +from swift.common import ondisk +from swift.common.ondisk import hash_path, normalize_timestamp from swift.common import ring from swift.obj import diskfile, replicator as object_replicator @@ -137,8 +138,8 @@ def _create_test_ring(path): class TestObjectReplicator(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' # Setup a test ring (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 17f82e8a19..638ccb5793 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -32,9 +32,10 @@ from test.unit import FakeLogger from test.unit import connect_tcp, readuntil2crlfs from swift.obj import server as object_server from swift.obj import diskfile -from swift.common import utils -from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ - NullLogger, storage_directory, public, replication +from swift.common.utils import mkdirs, NullLogger, public, replication +from swift.common.ondisk import hash_path, normalize_timestamp, \ + storage_directory +from swift.common import ondisk from swift.common import constraints from eventlet import tpool from swift.common.swob import Request, HeaderKeyDict @@ -45,8 +46,8 @@ class TestObjectController(unittest.TestCase): def setUp(self): """Set up for testing swift.object.server.ObjectController""" - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = 'startcap' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = 'startcap' self.testdir = \ os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController') mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) @@ -1904,8 +1905,8 @@ class TestObjectController(unittest.TestCase): 'x-trans-id': '-'})}) def test_async_update_saves_on_exception(self): - _prefix = utils.HASH_PATH_PREFIX - utils.HASH_PATH_PREFIX = '' + _prefix = ondisk.HASH_PATH_PREFIX + ondisk.HASH_PATH_PREFIX = '' def fake_http_connect(*args): raise Exception('test') @@ -1918,7 +1919,7 @@ class TestObjectController(unittest.TestCase): {'x-timestamp': '1', 'x-out': 'set'}, 'sda1') finally: object_server.http_connect = orig_http_connect - utils.HASH_PATH_PREFIX = _prefix + ondisk.HASH_PATH_PREFIX = _prefix self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', 'async_pending', 'a83', @@ -1928,8 +1929,8 @@ class TestObjectController(unittest.TestCase): 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) def test_async_update_saves_on_non_2xx(self): - _prefix = utils.HASH_PATH_PREFIX - utils.HASH_PATH_PREFIX = '' + _prefix = ondisk.HASH_PATH_PREFIX + ondisk.HASH_PATH_PREFIX = '' def fake_http_connect(status): @@ -1963,7 +1964,7 @@ class TestObjectController(unittest.TestCase): 'op': 'PUT'}) finally: object_server.http_connect = orig_http_connect - utils.HASH_PATH_PREFIX = _prefix + ondisk.HASH_PATH_PREFIX = _prefix def test_async_update_does_not_save_on_2xx(self): diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py index ebfac25bbb..97a07d059b 100644 --- a/test/unit/obj/test_updater.py +++ b/test/unit/obj/test_updater.py @@ -27,17 +27,17 @@ from eventlet import spawn, Timeout, listen from swift.obj import updater as object_updater, server as object_server from swift.obj.server import ASYNCDIR from swift.common.ring import RingData -from swift.common import utils -from swift.common.utils import hash_path, normalize_timestamp, mkdirs, \ - write_pickle +from swift.common.utils import mkdirs, write_pickle +from swift.common import ondisk +from swift.common.ondisk import hash_path, normalize_timestamp from test.unit import FakeLogger class TestObjectUpdater(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'object_updater') rmtree(self.testdir, ignore_errors=1) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 75fd79b3f4..6a32dd30a1 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -47,7 +47,9 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH from swift.common import utils -from swift.common.utils import mkdirs, normalize_timestamp, NullLogger +from swift.common.utils import mkdirs, NullLogger +from swift.common import ondisk +from swift.common.ondisk import normalize_timestamp from swift.common.wsgi import monkey_patch_mimetools from swift.proxy.controllers.obj import SegmentedIterable from swift.proxy.controllers.base import get_container_memcache_key, \ @@ -73,7 +75,7 @@ def request_init(self, *args, **kwargs): def setup(): - utils.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_SUFFIX = 'endcap' global _testdir, _test_servers, _test_sockets, \ _orig_container_listing_limit, _test_coros, _orig_SysLogHandler _orig_SysLogHandler = utils.SysLogHandler