Refactor common/utils methods to common/ondisk

Place all the methods related to on-disk layout and / or configuration
into a new common module that can be shared by the various modules
using the same on-disk layout.

Change-Id: I27ffd4665d5115ffdde649c48a4d18e12017e6a9
Signed-off-by: Peter Portante <peter.portante@redhat.com>

doc/source/misc.rst

@@ -33,6 +33,15 @@ Utils
     :members:
     :show-inheritance:
 
+.. _ondisk:
+
+OnDisk
+======
+
+.. automodule:: swift.common.ondisk
+    :members:
+    :show-inheritance:
+
 .. _common_tempauth:
 
 TempAuth

swift/account/auditor.py

@@ -21,8 +21,9 @@ from random import random
 import swift.common.db
 from swift.account import server as account_server
 from swift.account.backend import AccountBroker
-from swift.common.utils import get_logger, audit_location_generator, \
+from swift.common.utils import get_logger, config_true_value, \
-    config_true_value, dump_recon_cache, ratelimit_sleep
+    dump_recon_cache, ratelimit_sleep
+from swift.common.ondisk import audit_location_generator
 from swift.common.daemon import Daemon
 
 from eventlet import Timeout

swift/account/backend.py

@@ -25,7 +25,8 @@ import errno
 
 import sqlite3
 
-from swift.common.utils import normalize_timestamp, lock_parent_directory
+from swift.common.utils import lock_parent_directory
+from swift.common.ondisk import normalize_timestamp
 from swift.common.db import DatabaseBroker, DatabaseConnectionError, \
     PENDING_CAP, PICKLE_PROTOCOL, utf8encode
 

swift/account/server.py

@@ -28,9 +28,10 @@ from swift.account.utils import account_listing_response
 from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists
 from swift.common.request_helpers import get_param, get_listing_content_type, \
     split_and_validate_path
-from swift.common.utils import get_logger, hash_path, public, \
+from swift.common.utils import get_logger, public, config_true_value, \
-    normalize_timestamp, storage_directory, config_true_value, \
     json, timing_stats, replication
+from swift.common.ondisk import hash_path, normalize_timestamp, \
+    storage_directory
 from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
     check_mount, check_float, check_utf8
 from swift.common.db_replicator import ReplicatorRpc

swift/account/utils.py

@@ -17,7 +17,8 @@ import time
 from xml.sax import saxutils
 
 from swift.common.swob import HTTPOk, HTTPNoContent
-from swift.common.utils import json, normalize_timestamp
+from swift.common.utils import json
+from swift.common.ondisk import normalize_timestamp
 
 
 class FakeAccountBroker(object):

swift/common/daemon.py

@@ -22,6 +22,7 @@ from re import sub
 import eventlet.debug
 
 from swift.common import utils
+from swift.common import ondisk
 
 
 class Daemon(object):
@@ -41,7 +42,7 @@ class Daemon(object):
 
     def run(self, once=False, **kwargs):
         """Run the daemon"""
-        utils.validate_configuration()
+        ondisk.validate_configuration()
         utils.drop_privileges(self.conf.get('user', 'swift'))
         utils.capture_stdio(self.logger, **kwargs)
 

swift/common/db.py

@@ -30,8 +30,9 @@ from tempfile import mkstemp
 from eventlet import sleep, Timeout
 import sqlite3
 
-from swift.common.utils import json, normalize_timestamp, renamer, \
+from swift.common.utils import json, renamer, mkdirs, lock_parent_directory, \
-    mkdirs, lock_parent_directory, fallocate
+    fallocate
+from swift.common.ondisk import normalize_timestamp
 from swift.common.exceptions import LockTimeout
 
 

swift/common/db_replicator.py

@@ -30,9 +30,10 @@ import simplejson
 
 import swift.common.db
 from swift.common.direct_client import quote
-from swift.common.utils import get_logger, whataremyips, storage_directory, \
+from swift.common.utils import get_logger, whataremyips, renamer, mkdirs, \
-    renamer, mkdirs, lock_parent_directory, config_true_value, \
+    lock_parent_directory, config_true_value, unlink_older_than, \
-    unlink_older_than, dump_recon_cache, rsync_ip
+    dump_recon_cache, rsync_ip
+from swift.common.ondisk import storage_directory
 from swift.common import ring
 from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE
 from swift.common.bufferedhttp import BufferedHTTPConnection

swift/common/direct_client.py

@@ -27,7 +27,8 @@ from eventlet import sleep, Timeout
 
 from swift.common.bufferedhttp import http_connect
 from swiftclient import ClientException, json_loads
-from swift.common.utils import normalize_timestamp, FileLikeIter
+from swift.common.utils import FileLikeIter
+from swift.common.ondisk import normalize_timestamp
 from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
     is_success, is_server_error
 from swift.common.swob import HeaderKeyDict

swift/common/ondisk.py

@@ -0,0 +1,184 @@
+# Copyright (c) 2010-2013 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Methods & Attributes for shared 'on-disk' data layouts."""
+
+import os
+import sys
+import errno
+
+from hashlib import md5
+from random import shuffle
+from ConfigParser import ConfigParser, NoSectionError, NoOptionError
+
+from swift import gettext_ as _
+from swift.common.utils import listdir, quote
+
+# Used by hash_path to offer a bit more security when generating hashes for
+# paths. It simply appends this value to all paths; guessing the hash a path
+# will end up with would also require knowing this suffix.
+_hash_conf = ConfigParser()
+HASH_PATH_SUFFIX = ''
+HASH_PATH_PREFIX = ''
+if _hash_conf.read('/etc/swift/swift.conf'):
+    try:
+        HASH_PATH_SUFFIX = _hash_conf.get('swift-hash',
+                                          'swift_hash_path_suffix')
+    except (NoSectionError, NoOptionError):
+        pass
+    try:
+        HASH_PATH_PREFIX = _hash_conf.get('swift-hash',
+                                          'swift_hash_path_prefix')
+    except (NoSectionError, NoOptionError):
+        pass
+
+
+def validate_configuration():
+    if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX:
+        sys.exit("Error: [swift-hash]: both swift_hash_path_suffix "
+                 "and swift_hash_path_prefix are missing "
+                 "from /etc/swift/swift.conf")
+
+
+def hash_path(account, container=None, object=None, raw_digest=False):
+    """
+    Get the canonical hash for an account/container/object
+
+    :param account: Account
+    :param container: Container
+    :param object: Object
+    :param raw_digest: If True, return the raw version rather than a hex digest
+    :returns: hash string
+    """
+    if object and not container:
+        raise ValueError('container is required if object is provided')
+    paths = [account]
+    if container:
+        paths.append(container)
+    if object:
+        paths.append(object)
+    if raw_digest:
+        return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths)
+                   + HASH_PATH_SUFFIX).digest()
+    else:
+        return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths)
+                   + HASH_PATH_SUFFIX).hexdigest()
+
+
+def normalize_timestamp(timestamp):
+    """
+    Format a timestamp (string or numeric) into a standardized
+    xxxxxxxxxx.xxxxx (10.5) format.
+
+    Note that timestamps using values greater than or equal to November 20th,
+    2286 at 17:46 UTC will use 11 digits to represent the number of
+    seconds.
+
+    :param timestamp: unix timestamp
+    :returns: normalized timestamp as a string
+    """
+    return "%016.05f" % (float(timestamp))
+
+
+def validate_device_partition(device, partition):
+    """
+    Validate that a device and a partition are valid and won't lead to
+    directory traversal when used.
+
+    :param device: device to validate
+    :param partition: partition to validate
+    :raises: ValueError if given an invalid device or partition
+    """
+    invalid_device = False
+    invalid_partition = False
+    if not device or '/' in device or device in ['.', '..']:
+        invalid_device = True
+    if not partition or '/' in partition or partition in ['.', '..']:
+        invalid_partition = True
+
+    if invalid_device:
+        raise ValueError('Invalid device: %s' % quote(device or ''))
+    elif invalid_partition:
+        raise ValueError('Invalid partition: %s' % quote(partition or ''))
+
+
+def storage_directory(datadir, partition, name_hash):
+    """
+    Get the storage directory
+
+    :param datadir: Base data directory
+    :param partition: Partition
+    :param name_hash: Account, container or object name hash
+    :returns: Storage directory
+    """
+    return os.path.join(datadir, str(partition), name_hash[-3:], name_hash)
+
+
+def audit_location_generator(devices, datadir, suffix='',
+                             mount_check=True, logger=None):
+    '''
+    Given a devices path and a data directory, yield (path, device,
+    partition) for all files in that directory
+
+    :param devices: parent directory of the devices to be audited
+    :param datadir: a directory located under self.devices. This should be
+                    one of the DATADIR constants defined in the account,
+                    container, and object servers.
+    :param suffix: path name suffix required for all names returned
+    :param mount_check: Flag to check if a mount check should be performed
+                    on devices
+    :param logger: a logger object
+    '''
+    device_dir = listdir(devices)
+    # randomize devices in case of process restart before sweep completed
+    shuffle(device_dir)
+    for device in device_dir:
+        if mount_check and not \
+                os.path.ismount(os.path.join(devices, device)):
+            if logger:
+                logger.debug(
+                    _('Skipping %s as it is not mounted'), device)
+            continue
+        datadir_path = os.path.join(devices, device, datadir)
+        partitions = listdir(datadir_path)
+        for partition in partitions:
+            part_path = os.path.join(datadir_path, partition)
+            try:
+                suffixes = listdir(part_path)
+            except OSError as e:
+                if e.errno != errno.ENOTDIR:
+                    raise
+                continue
+            for asuffix in suffixes:
+                suff_path = os.path.join(part_path, asuffix)
+                try:
+                    hashes = listdir(suff_path)
+                except OSError as e:
+                    if e.errno != errno.ENOTDIR:
+                        raise
+                    continue
+                for hsh in hashes:
+                    hash_path = os.path.join(suff_path, hsh)
+                    try:
+                        files = sorted(listdir(hash_path), reverse=True)
+                    except OSError as e:
+                        if e.errno != errno.ENOTDIR:
+                            raise
+                        continue
+                    for fname in files:
+                        if suffix and not fname.endswith(suffix):
+                            continue
+                        path = os.path.join(hash_path, fname)
+                        yield path, device, partition

swift/common/request_helpers.py

@@ -22,7 +22,8 @@ from swob in here without creating circular imports.
 
 from swift.common.constraints import FORMAT2CONTENT_TYPE
 from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable
-from swift.common.utils import split_path, validate_device_partition
+from swift.common.ondisk import validate_device_partition
+from swift.common.utils import split_path
 from urllib import unquote
 
 

swift/common/ring/ring.py

@@ -25,7 +25,8 @@ from io import BufferedReader
 from hashlib import md5
 from itertools import chain
 
-from swift.common.utils import hash_path, validate_configuration, json
+from swift.common.utils import json
+from swift.common.ondisk import hash_path, validate_configuration
 from swift.common.ring.utils import tiers_for_dev
 
 
@@ -130,7 +131,7 @@ class Ring(object):
     """
 
     def __init__(self, serialized_path, reload_time=15, ring_name=None):
-        # can't use the ring unless HASH_PATH_SUFFIX is set
+        # Can't use the ring unless the on-disk configuration is valid
         validate_configuration()
         if ring_name:
             self.serialized_path = os.path.join(serialized_path,

swift/common/utils.py

@@ -26,14 +26,12 @@ import threading as stdlib_threading
 import time
 import uuid
 import functools
-from hashlib import md5
+from random import random
-from random import random, shuffle
 from urllib import quote as _quote
 from contextlib import contextmanager, closing
 import ctypes
 import ctypes.util
-from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \
+from ConfigParser import ConfigParser, RawConfigParser
-    RawConfigParser
 from optparse import OptionParser
 from Queue import Queue, Empty
 from tempfile import mkstemp, NamedTemporaryFile
@@ -80,24 +78,6 @@ _posix_fadvise = None
 # available being at or below this amount, in bytes.
 FALLOCATE_RESERVE = 0
 
-# Used by hash_path to offer a bit more security when generating hashes for
-# paths. It simply appends this value to all paths; guessing the hash a path
-# will end up with would also require knowing this suffix.
-hash_conf = ConfigParser()
-HASH_PATH_SUFFIX = ''
-HASH_PATH_PREFIX = ''
-if hash_conf.read('/etc/swift/swift.conf'):
-    try:
-        HASH_PATH_SUFFIX = hash_conf.get('swift-hash',
-                                         'swift_hash_path_suffix')
-    except (NoSectionError, NoOptionError):
-        pass
-    try:
-        HASH_PATH_PREFIX = hash_conf.get('swift-hash',
-                                         'swift_hash_path_prefix')
-    except (NoSectionError, NoOptionError):
-        pass
-
 
 def backward(f, blocksize=4096):
     """
@@ -164,13 +144,6 @@ def noop_libc_function(*args):
     return 0
 
 
-def validate_configuration():
-    if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX:
-        sys.exit("Error: [swift-hash]: both swift_hash_path_suffix "
-                 "and swift_hash_path_prefix are missing "
-                 "from /etc/swift/swift.conf")
-
-
 def load_libc_function(func_name, log_error=True):
     """
     Attempt to find the function in libc, otherwise return a no-op func.
@@ -422,21 +395,6 @@ def drop_buffer_cache(fd, offset, length):
                      % (fd, offset, length, ret))
 
 
-def normalize_timestamp(timestamp):
-    """
-    Format a timestamp (string or numeric) into a standardized
-    xxxxxxxxxx.xxxxx (10.5) format.
-
-    Note that timestamps using values greater than or equal to November 20th,
-    2286 at 17:46 UTC will use 11 digits to represent the number of
-    seconds.
-
-    :param timestamp: unix timestamp
-    :returns: normalized timestamp as a string
-    """
-    return "%016.05f" % (float(timestamp))
-
-
 def mkdirs(path):
     """
     Ensures the path is a directory or makes it if not. Errors if the path
@@ -515,28 +473,6 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False):
     return segs
 
 
-def validate_device_partition(device, partition):
-    """
-    Validate that a device and a partition are valid and won't lead to
-    directory traversal when used.
-
-    :param device: device to validate
-    :param partition: partition to validate
-    :raises: ValueError if given an invalid device or partition
-    """
-    invalid_device = False
-    invalid_partition = False
-    if not device or '/' in device or device in ['.', '..']:
-        invalid_device = True
-    if not partition or '/' in partition or partition in ['.', '..']:
-        invalid_partition = True
-
-    if invalid_device:
-        raise ValueError('Invalid device: %s' % quote(device or ''))
-    elif invalid_partition:
-        raise ValueError('Invalid partition: %s' % quote(partition or ''))
-
-
 class GreenthreadSafeIterator(object):
     """
     Wrap an iterator to ensure that only one greenthread is inside its next()
@@ -1140,43 +1076,6 @@ def whataremyips():
     return addresses
 
 
-def storage_directory(datadir, partition, name_hash):
-    """
-    Get the storage directory
-
-    :param datadir: Base data directory
-    :param partition: Partition
-    :param name_hash: Account, container or object name hash
-    :returns: Storage directory
-    """
-    return os.path.join(datadir, str(partition), name_hash[-3:], name_hash)
-
-
-def hash_path(account, container=None, object=None, raw_digest=False):
-    """
-    Get the canonical hash for an account/container/object
-
-    :param account: Account
-    :param container: Container
-    :param object: Object
-    :param raw_digest: If True, return the raw version rather than a hex digest
-    :returns: hash string
-    """
-    if object and not container:
-        raise ValueError('container is required if object is provided')
-    paths = [account]
-    if container:
-        paths.append(container)
-    if object:
-        paths.append(object)
-    if raw_digest:
-        return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths)
-                   + HASH_PATH_SUFFIX).digest()
-    else:
-        return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths)
-                   + HASH_PATH_SUFFIX).hexdigest()
-
-
 @contextmanager
 def lock_path(directory, timeout=10):
     """
@@ -1491,64 +1390,6 @@ def remove_file(path):
         pass
 
 
-def audit_location_generator(devices, datadir, suffix='',
-                             mount_check=True, logger=None):
-    '''
-    Given a devices path and a data directory, yield (path, device,
-    partition) for all files in that directory
-
-    :param devices: parent directory of the devices to be audited
-    :param datadir: a directory located under self.devices. This should be
-                    one of the DATADIR constants defined in the account,
-                    container, and object servers.
-    :param suffix: path name suffix required for all names returned
-    :param mount_check: Flag to check if a mount check should be performed
-                    on devices
-    :param logger: a logger object
-    '''
-    device_dir = listdir(devices)
-    # randomize devices in case of process restart before sweep completed
-    shuffle(device_dir)
-    for device in device_dir:
-        if mount_check and not \
-                os.path.ismount(os.path.join(devices, device)):
-            if logger:
-                logger.debug(
-                    _('Skipping %s as it is not mounted'), device)
-            continue
-        datadir_path = os.path.join(devices, device, datadir)
-        partitions = listdir(datadir_path)
-        for partition in partitions:
-            part_path = os.path.join(datadir_path, partition)
-            try:
-                suffixes = listdir(part_path)
-            except OSError as e:
-                if e.errno != errno.ENOTDIR:
-                    raise
-                continue
-            for asuffix in suffixes:
-                suff_path = os.path.join(part_path, asuffix)
-                try:
-                    hashes = listdir(suff_path)
-                except OSError as e:
-                    if e.errno != errno.ENOTDIR:
-                        raise
-                    continue
-                for hsh in hashes:
-                    hash_path = os.path.join(suff_path, hsh)
-                    try:
-                        files = sorted(listdir(hash_path), reverse=True)
-                    except OSError as e:
-                        if e.errno != errno.ENOTDIR:
-                            raise
-                        continue
-                    for fname in files:
-                        if suffix and not fname.endswith(suffix):
-                            continue
-                        path = os.path.join(hash_path, fname)
-                        yield path, device, partition
-
-
 def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5):
     '''
     Will eventlet.sleep() for the appropriate time so that the max_rate

swift/common/wsgi.py

@@ -35,7 +35,8 @@ from swift.common import utils
 from swift.common.swob import Request
 from swift.common.utils import capture_stdio, disable_fallocate, \
     drop_privileges, get_logger, NullLogger, config_true_value, \
-    validate_configuration, get_hub, config_auto_int_value
+    get_hub, config_auto_int_value
+from swift.common.ondisk import validate_configuration
 
 try:
     import multiprocessing

swift/container/auditor.py

@@ -23,8 +23,9 @@ from eventlet import Timeout
 import swift.common.db
 from swift.container import server as container_server
 from swift.container.backend import ContainerBroker
-from swift.common.utils import get_logger, audit_location_generator, \
+from swift.common.utils import get_logger, config_true_value, \
-    config_true_value, dump_recon_cache, ratelimit_sleep
+    dump_recon_cache, ratelimit_sleep
+from swift.common.ondisk import audit_location_generator
 from swift.common.daemon import Daemon
 
 

swift/container/backend.py

@@ -25,7 +25,8 @@ import errno
 
 import sqlite3
 
-from swift.common.utils import normalize_timestamp, lock_parent_directory
+from swift.common.utils import lock_parent_directory
+from swift.common.ondisk import normalize_timestamp
 from swift.common.db import DatabaseBroker, DatabaseConnectionError, \
     PENDING_CAP, PICKLE_PROTOCOL, utf8encode
 

swift/container/server.py

@@ -29,9 +29,10 @@ from swift.container.backend import ContainerBroker
 from swift.common.db import DatabaseAlreadyExists
 from swift.common.request_helpers import get_param, get_listing_content_type, \
     split_and_validate_path
-from swift.common.utils import get_logger, hash_path, public, \
+from swift.common.utils import get_logger, public, validate_sync_to, \
-    normalize_timestamp, storage_directory, validate_sync_to, \
     config_true_value, json, timing_stats, replication, parse_content_type
+from swift.common.ondisk import hash_path, normalize_timestamp, \
+    storage_directory
 from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
     check_mount, check_float, check_utf8
 from swift.common.bufferedhttp import http_connect

swift/container/sync.py

@@ -27,8 +27,9 @@ from swiftclient import ClientException, delete_object, put_object, \
 from swift.container.backend import ContainerBroker
 from swift.common.direct_client import direct_get_object
 from swift.common.ring import Ring
-from swift.common.utils import audit_location_generator, get_logger, \
+from swift.common.utils import get_logger, config_true_value, \
-    hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter
+    validate_sync_to, whataremyips, FileLikeIter
+from swift.common.ondisk import audit_location_generator, hash_path
 from swift.common.daemon import Daemon
 from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND
 

swift/obj/auditor.py

@@ -21,8 +21,9 @@ from eventlet import Timeout
 
 from swift.obj import diskfile
 from swift.obj import server as object_server
-from swift.common.utils import get_logger, audit_location_generator, \
+from swift.common.utils import get_logger, ratelimit_sleep, \
-    ratelimit_sleep, config_true_value, dump_recon_cache, list_from_csv, json
+    config_true_value, dump_recon_cache, list_from_csv, json
+from swift.common.ondisk import audit_location_generator
 from swift.common.exceptions import AuditException, DiskFileError, \
     DiskFileNotExist
 from swift.common.daemon import Daemon

swift/obj/diskfile.py

@@ -33,9 +33,10 @@ from eventlet import Timeout
 
 from swift import gettext_ as _
 from swift.common.constraints import check_mount
-from swift.common.utils import mkdirs, normalize_timestamp, \
+from swift.common.utils import mkdirs, renamer, fallocate, fsync, fdatasync, \
-    storage_directory, hash_path, renamer, fallocate, fsync, \
+    drop_buffer_cache, ThreadPool, lock_path, write_pickle
-    fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle
+from swift.common.ondisk import hash_path, normalize_timestamp, \
+    storage_directory
 from swift.common.exceptions import DiskFileError, DiskFileNotExist, \
     DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \
     PathNotDir, DiskFileNotOpenError

swift/obj/server.py

@@ -27,9 +27,9 @@ from hashlib import md5
 
 from eventlet import sleep, Timeout
 
-from swift.common.utils import mkdirs, normalize_timestamp, public, \
+from swift.common.utils import mkdirs, public, get_logger, write_pickle, \
-    hash_path, get_logger, write_pickle, config_true_value, timing_stats, \
+    config_true_value, timing_stats, ThreadPool, replication
-    ThreadPool, replication
+from swift.common.ondisk import normalize_timestamp, hash_path
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_object_creation, check_mount, \
     check_float, check_utf8

swift/proxy/controllers/base.py

@@ -37,9 +37,9 @@ from eventlet.queue import Queue, Empty, Full
 from eventlet.timeout import Timeout
 
 from swift.common.wsgi import make_pre_authed_env
-from swift.common.utils import normalize_timestamp, config_true_value, \
+from swift.common.utils import config_true_value, public, split_path, \
-    public, split_path, list_from_csv, GreenthreadSafeIterator, \
+    list_from_csv, GreenthreadSafeIterator, quorum_size
-    quorum_size
+from swift.common.ondisk import normalize_timestamp
 from swift.common.bufferedhttp import http_connect
 from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout
 from swift.common.http import is_informational, is_success, is_redirection, \

swift/proxy/controllers/obj.py

@@ -37,9 +37,9 @@ from eventlet import sleep, GreenPile
 from eventlet.queue import Queue
 from eventlet.timeout import Timeout
 
-from swift.common.utils import ContextPool, normalize_timestamp, \
+from swift.common.utils import ContextPool, config_true_value, public, json, \
-    config_true_value, public, json, csv_append, GreenthreadSafeIterator, \
+    csv_append, GreenthreadSafeIterator, quorum_size
-    quorum_size
+from swift.common.ondisk import normalize_timestamp
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_metadata, check_object_creation, \
     CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE, MAX_BUFFERED_SLO_SEGMENTS

test/unit/account/test_backend.py

@@ -22,7 +22,7 @@ from time import sleep, time
 from uuid import uuid4
 
 from swift.account.backend import AccountBroker
-from swift.common.utils import normalize_timestamp
+from swift.common.ondisk import normalize_timestamp
 
 
 class TestAccountBroker(unittest.TestCase):

test/unit/account/test_reaper.py

@@ -25,7 +25,7 @@ from contextlib import nested
 
 from swift.account import reaper
 from swift.account.server import DATADIR
-from swift.common.utils import normalize_timestamp
+from swift.common.ondisk import normalize_timestamp
 from swift.common.direct_client import ClientException
 
 

test/unit/account/test_server.py

@@ -25,7 +25,8 @@ import xml.dom.minidom
 
 from swift.common.swob import Request
 from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT
-from swift.common.utils import normalize_timestamp, replication, public
+from swift.common.utils import replication, public
+from swift.common.ondisk import normalize_timestamp
 
 
 class TestAccountController(unittest.TestCase):

test/unit/common/middleware/test_list_endpoints.py

@@ -18,7 +18,7 @@ import unittest
 from shutil import rmtree
 
 import os
-from swift.common import ring, utils
+from swift.common import ring, ondisk
 from swift.common.utils import json
 from swift.common.swob import Request, Response
 from swift.common.middleware import list_endpoints
@@ -35,8 +35,8 @@ def start_response(*args):
 
 class TestListEndpoints(unittest.TestCase):
     def setUp(self):
-        utils.HASH_PATH_SUFFIX = 'endcap'
+        ondisk.HASH_PATH_SUFFIX = 'endcap'
-        utils.HASH_PATH_PREFIX = ''
+        ondisk.HASH_PATH_PREFIX = ''
         self.testdir = os.path.join(os.path.dirname(__file__), 'ring')
         rmtree(self.testdir, ignore_errors=1)
         os.mkdir(self.testdir)

test/unit/common/ring/test_ring.py

@@ -23,7 +23,7 @@ from gzip import GzipFile
 from shutil import rmtree
 from time import sleep, time
 
-from swift.common import ring, utils
+from swift.common import ring, ondisk
 
 
 class TestRingData(unittest.TestCase):
@@ -99,8 +99,8 @@ class TestRingData(unittest.TestCase):
 class TestRing(unittest.TestCase):
 
     def setUp(self):
-        utils.HASH_PATH_SUFFIX = 'endcap'
+        ondisk.HASH_PATH_SUFFIX = 'endcap'
-        utils.HASH_PATH_PREFIX = ''
+        ondisk.HASH_PATH_PREFIX = ''
         self.testdir = os.path.join(os.path.dirname(__file__), 'ring')
         rmtree(self.testdir, ignore_errors=1)
         os.mkdir(self.testdir)
@@ -146,15 +146,15 @@ class TestRing(unittest.TestCase):
         self.assertEquals(self.ring.reload_time, self.intended_reload_time)
         self.assertEquals(self.ring.serialized_path, self.testgz)
         # test invalid endcap
-        _orig_hash_path_suffix = utils.HASH_PATH_SUFFIX
+        _orig_hash_path_suffix = ondisk.HASH_PATH_SUFFIX
-        _orig_hash_path_prefix = utils.HASH_PATH_PREFIX
+        _orig_hash_path_prefix = ondisk.HASH_PATH_PREFIX
         try:
-            utils.HASH_PATH_SUFFIX = ''
+            ondisk.HASH_PATH_SUFFIX = ''
-            utils.HASH_PATH_PREFIX = ''
+            ondisk.HASH_PATH_PREFIX = ''
             self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever')
         finally:
-            utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix
+            ondisk.HASH_PATH_SUFFIX = _orig_hash_path_suffix
-            utils.HASH_PATH_PREFIX = _orig_hash_path_prefix
+            ondisk.HASH_PATH_PREFIX = _orig_hash_path_prefix
 
     def test_has_changed(self):
         self.assertEquals(self.ring.has_changed(), False)

test/unit/common/test_daemon.py

@@ -23,7 +23,7 @@ from StringIO import StringIO
 from test.unit import tmpfile
 from mock import patch
 
-from swift.common import daemon, utils
+from swift.common import daemon, utils, ondisk
 
 
 class MyDaemon(daemon.Daemon):
@@ -63,8 +63,8 @@ class TestDaemon(unittest.TestCase):
 class TestRunDaemon(unittest.TestCase):
 
     def setUp(self):
-        utils.HASH_PATH_SUFFIX = 'endcap'
+        ondisk.HASH_PATH_SUFFIX = 'endcap'
-        utils.HASH_PATH_PREFIX = 'startcap'
+        ondisk.HASH_PATH_PREFIX = 'startcap'
         utils.drop_privileges = lambda *args: None
         utils.capture_stdio = lambda *args: None
 

test/unit/common/test_db.py

@@ -28,7 +28,7 @@ from mock import patch
 import swift.common.db
 from swift.common.db import chexor, dict_factory, get_db_connection, \
     DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists
-from swift.common.utils import normalize_timestamp
+from swift.common.ondisk import normalize_timestamp
 from swift.common.exceptions import LockTimeout
 
 

test/unit/common/test_db_replicator.py

@@ -26,7 +26,7 @@ import mock
 import simplejson
 
 from swift.common import db_replicator
-from swift.common.utils import normalize_timestamp
+from swift.common.ondisk import normalize_timestamp
 from swift.container import server as container_server
 from swift.common.exceptions import DriveNotMounted
 

test/unit/common/test_ondisk.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2010-2013 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for swift.common.ondisk"""
+
+from __future__ import with_statement
+from test.unit import temptree
+
+import os
+
+import unittest
+
+from swift.common import ondisk
+
+
+class TestOndisk(unittest.TestCase):
+    """Tests for swift.common.ondisk"""
+
+    def setUp(self):
+        ondisk.HASH_PATH_SUFFIX = 'endcap'
+        ondisk.HASH_PATH_PREFIX = 'startcap'
+
+    def test_normalize_timestamp(self):
+        # Test swift.common.ondisk.normalize_timestamp
+        self.assertEquals(ondisk.normalize_timestamp('1253327593.48174'),
+                          "1253327593.48174")
+        self.assertEquals(ondisk.normalize_timestamp(1253327593.48174),
+                          "1253327593.48174")
+        self.assertEquals(ondisk.normalize_timestamp('1253327593.48'),
+                          "1253327593.48000")
+        self.assertEquals(ondisk.normalize_timestamp(1253327593.48),
+                          "1253327593.48000")
+        self.assertEquals(ondisk.normalize_timestamp('253327593.48'),
+                          "0253327593.48000")
+        self.assertEquals(ondisk.normalize_timestamp(253327593.48),
+                          "0253327593.48000")
+        self.assertEquals(ondisk.normalize_timestamp('1253327593'),
+                          "1253327593.00000")
+        self.assertEquals(ondisk.normalize_timestamp(1253327593),
+                          "1253327593.00000")
+        self.assertRaises(ValueError, ondisk.normalize_timestamp, '')
+        self.assertRaises(ValueError, ondisk.normalize_timestamp, 'abc')
+
+    def test_validate_device_partition(self):
+        # Test swift.common.ondisk.validate_device_partition
+        ondisk.validate_device_partition('foo', 'bar')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, '', '')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, '', 'foo')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, 'foo', '')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, 'foo/bar', 'foo')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, 'foo', 'foo/bar')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, '.', 'foo')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, '..', 'foo')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, 'foo', '.')
+        self.assertRaises(ValueError,
+                          ondisk.validate_device_partition, 'foo', '..')
+        try:
+            ondisk.validate_device_partition('o\nn e', 'foo')
+        except ValueError as err:
+            self.assertEquals(str(err), 'Invalid device: o%0An%20e')
+        try:
+            ondisk.validate_device_partition('foo', 'o\nn e')
+        except ValueError as err:
+            self.assertEquals(str(err), 'Invalid partition: o%0An%20e')
+
+    def test_storage_directory(self):
+        self.assertEquals(ondisk.storage_directory('objects', '1', 'ABCDEF'),
+                          'objects/1/DEF/ABCDEF')
+
+    def test_hash_path(self):
+        _prefix = ondisk.HASH_PATH_PREFIX
+        ondisk.HASH_PATH_PREFIX = ''
+        # Yes, these tests are deliberately very fragile. We want to make sure
+        # that if someones changes the results hash_path produces, they know it
+        try:
+            self.assertEquals(ondisk.hash_path('a'),
+                              '1c84525acb02107ea475dcd3d09c2c58')
+            self.assertEquals(ondisk.hash_path('a', 'c'),
+                              '33379ecb053aa5c9e356c68997cbb59e')
+            self.assertEquals(ondisk.hash_path('a', 'c', 'o'),
+                              '06fbf0b514e5199dfc4e00f42eb5ea83')
+            self.assertEquals(
+                ondisk.hash_path('a', 'c', 'o', raw_digest=False),
+                '06fbf0b514e5199dfc4e00f42eb5ea83')
+            self.assertEquals(ondisk.hash_path('a', 'c', 'o', raw_digest=True),
+                              '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN'
+                              '\x00\xf4.\xb5\xea\x83')
+            self.assertRaises(ValueError, ondisk.hash_path, 'a', object='o')
+            ondisk.HASH_PATH_PREFIX = 'abcdef'
+            self.assertEquals(
+                ondisk.hash_path('a', 'c', 'o', raw_digest=False),
+                '363f9b535bfb7d17a43a46a358afca0e')
+        finally:
+            ondisk.HASH_PATH_PREFIX = _prefix
+
+
+class TestAuditLocationGenerator(unittest.TestCase):
+    def test_non_dir_contents(self):
+        with temptree([]) as tmpdir:
+            data = os.path.join(tmpdir, "drive", "data")
+            os.makedirs(data)
+            with open(os.path.join(data, "partition1"), "w"):
+                pass
+            partition = os.path.join(data, "partition2")
+            os.makedirs(partition)
+            with open(os.path.join(partition, "suffix1"), "w"):
+                pass
+            suffix = os.path.join(partition, "suffix2")
+            os.makedirs(suffix)
+            with open(os.path.join(suffix, "hash1"), "w"):
+                pass
+            locations = ondisk.audit_location_generator(
+                tmpdir, "data", mount_check=False
+            )
+            self.assertEqual(list(locations), [])
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/test_utils.py

@@ -132,31 +132,6 @@ def reset_loggers():
 class TestUtils(unittest.TestCase):
     """Tests for swift.common.utils """
 
-    def setUp(self):
-        utils.HASH_PATH_SUFFIX = 'endcap'
-        utils.HASH_PATH_PREFIX = 'startcap'
-
-    def test_normalize_timestamp(self):
-        # Test swift.common.utils.normalize_timestamp
-        self.assertEquals(utils.normalize_timestamp('1253327593.48174'),
-                          "1253327593.48174")
-        self.assertEquals(utils.normalize_timestamp(1253327593.48174),
-                          "1253327593.48174")
-        self.assertEquals(utils.normalize_timestamp('1253327593.48'),
-                          "1253327593.48000")
-        self.assertEquals(utils.normalize_timestamp(1253327593.48),
-                          "1253327593.48000")
-        self.assertEquals(utils.normalize_timestamp('253327593.48'),
-                          "0253327593.48000")
-        self.assertEquals(utils.normalize_timestamp(253327593.48),
-                          "0253327593.48000")
-        self.assertEquals(utils.normalize_timestamp('1253327593'),
-                          "1253327593.00000")
-        self.assertEquals(utils.normalize_timestamp(1253327593),
-                          "1253327593.00000")
-        self.assertRaises(ValueError, utils.normalize_timestamp, '')
-        self.assertRaises(ValueError, utils.normalize_timestamp, 'abc')
-
     def test_backwards(self):
         # Test swift.common.utils.backward
 
@@ -248,36 +223,6 @@ class TestUtils(unittest.TestCase):
         except ValueError as err:
             self.assertEquals(str(err), 'Invalid path: o%0An%20e')
 
-    def test_validate_device_partition(self):
-        # Test swift.common.utils.validate_device_partition
-        utils.validate_device_partition('foo', 'bar')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, '', '')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, '', 'foo')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, 'foo', '')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, 'foo/bar', 'foo')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, 'foo', 'foo/bar')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, '.', 'foo')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, '..', 'foo')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, 'foo', '.')
-        self.assertRaises(ValueError,
-                          utils.validate_device_partition, 'foo', '..')
-        try:
-            utils.validate_device_partition('o\nn e', 'foo')
-        except ValueError as err:
-            self.assertEquals(str(err), 'Invalid device: o%0An%20e')
-        try:
-            utils.validate_device_partition('foo', 'o\nn e')
-        except ValueError as err:
-            self.assertEquals(str(err), 'Invalid partition: o%0An%20e')
-
     def test_NullLogger(self):
         # Test swift.common.utils.NullLogger
         sio = StringIO()
@@ -637,10 +582,6 @@ class TestUtils(unittest.TestCase):
             logger.logger.removeHandler(handler)
             reset_loggers()
 
-    def test_storage_directory(self):
-        self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'),
-                          'objects/1/DEF/ABCDEF')
-
     def test_whataremyips(self):
         myips = utils.whataremyips()
         self.assert_(len(myips) > 1)
@@ -676,30 +617,6 @@ class TestUtils(unittest.TestCase):
             self.assertEquals(len(myips), 1)
             self.assertEquals(myips[0], test_ipv6_address)
 
-    def test_hash_path(self):
-        _prefix = utils.HASH_PATH_PREFIX
-        utils.HASH_PATH_PREFIX = ''
-        # Yes, these tests are deliberately very fragile. We want to make sure
-        # that if someones changes the results hash_path produces, they know it
-        try:
-            self.assertEquals(utils.hash_path('a'),
-                              '1c84525acb02107ea475dcd3d09c2c58')
-            self.assertEquals(utils.hash_path('a', 'c'),
-                              '33379ecb053aa5c9e356c68997cbb59e')
-            self.assertEquals(utils.hash_path('a', 'c', 'o'),
-                              '06fbf0b514e5199dfc4e00f42eb5ea83')
-            self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False),
-                              '06fbf0b514e5199dfc4e00f42eb5ea83')
-            self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True),
-                              '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN'
-                              '\x00\xf4.\xb5\xea\x83')
-            self.assertRaises(ValueError, utils.hash_path, 'a', object='o')
-            utils.HASH_PATH_PREFIX = 'abcdef'
-            self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False),
-                              '363f9b535bfb7d17a43a46a358afca0e')
-        finally:
-            utils.HASH_PATH_PREFIX = _prefix
-
     def test_load_libc_function(self):
         self.assert_(callable(
             utils.load_libc_function('printf')))
@@ -2401,26 +2318,5 @@ class TestThreadpool(unittest.TestCase):
         self.assertTrue(caught)
 
 
-class TestAuditLocationGenerator(unittest.TestCase):
-    def test_non_dir_contents(self):
-        with temptree([]) as tmpdir:
-            data = os.path.join(tmpdir, "drive", "data")
-            os.makedirs(data)
-            with open(os.path.join(data, "partition1"), "w"):
-                pass
-            partition = os.path.join(data, "partition2")
-            os.makedirs(partition)
-            with open(os.path.join(partition, "suffix1"), "w"):
-                pass
-            suffix = os.path.join(partition, "suffix2")
-            os.makedirs(suffix)
-            with open(os.path.join(suffix, "hash1"), "w"):
-                pass
-            locations = utils.audit_location_generator(
-                tmpdir, "data", mount_check=False
-            )
-            self.assertEqual(list(locations), [])
-
-
 if __name__ == '__main__':
     unittest.main()

test/unit/container/test_backend.py

@@ -22,7 +22,7 @@ from time import sleep, time
 from uuid import uuid4
 
 from swift.container.backend import ContainerBroker
-from swift.common.utils import normalize_timestamp
+from swift.common.ondisk import normalize_timestamp
 
 
 class TestContainerBroker(unittest.TestCase):

test/unit/container/test_replicator.py

@@ -15,7 +15,7 @@
 
 import unittest
 from swift.container import replicator
-from swift.common.utils import normalize_timestamp
+from swift.common.ondisk import normalize_timestamp
 
 
 class TestReplicator(unittest.TestCase):

test/unit/container/test_server.py

@@ -29,7 +29,8 @@ import simplejson
 from swift.common.swob import Request, HeaderKeyDict
 import swift.container
 from swift.container import server as container_server
-from swift.common.utils import normalize_timestamp, mkdirs, public, replication
+from swift.common.utils import mkdirs, public, replication
+from swift.common.ondisk import normalize_timestamp
 from test.unit import fake_http_connect
 
 

test/unit/container/test_updater.py

@@ -28,7 +28,7 @@ from swift.container import updater as container_updater
 from swift.container import server as container_server
 from swift.container.backend import ContainerBroker
 from swift.common.ring import RingData
-from swift.common.utils import normalize_timestamp
+from swift.common.ondisk import normalize_timestamp
 
 
 class TestContainerUpdater(unittest.TestCase):

test/unit/obj/test_auditor.py

@@ -25,7 +25,8 @@ from test.unit import FakeLogger
 from swift.obj import auditor
 from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash
 from swift.obj.server import DATADIR
-from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
+from swift.common.utils import mkdirs
+from swift.common.ondisk import hash_path, normalize_timestamp, \
     storage_directory
 
 

test/unit/obj/test_diskfile.py

@@ -36,8 +36,9 @@ from eventlet import tpool
 from test.unit import FakeLogger, mock as unit_mock
 from test.unit import _setxattr as setxattr
 from swift.obj import diskfile
-from swift.common import utils
+from swift.common import ondisk
-from swift.common.utils import hash_path, mkdirs, normalize_timestamp
+from swift.common.utils import mkdirs
+from swift.common.ondisk import hash_path, normalize_timestamp
 from swift.common import ring
 from swift.common.exceptions import DiskFileNotExist, DiskFileDeviceUnavailable
 
@@ -72,8 +73,8 @@ def _create_test_ring(path):
 class TestDiskFileModuleMethods(unittest.TestCase):
 
     def setUp(self):
-        utils.HASH_PATH_SUFFIX = 'endcap'
+        ondisk.HASH_PATH_SUFFIX = 'endcap'
-        utils.HASH_PATH_PREFIX = ''
+        ondisk.HASH_PATH_PREFIX = ''
         # Setup a test ring (stolen from common/test_ring.py)
         self.testdir = tempfile.mkdtemp()
         self.devices = os.path.join(self.testdir, 'node')

test/unit/obj/test_replicator.py

@@ -29,8 +29,9 @@ from eventlet.green import subprocess
 from eventlet import Timeout, tpool
 
 from test.unit import FakeLogger
-from swift.common import utils
+from swift.common.utils import mkdirs
-from swift.common.utils import hash_path, mkdirs, normalize_timestamp
+from swift.common import ondisk
+from swift.common.ondisk import hash_path, normalize_timestamp
 from swift.common import ring
 from swift.obj import diskfile, replicator as object_replicator
 
@@ -137,8 +138,8 @@ def _create_test_ring(path):
 class TestObjectReplicator(unittest.TestCase):
 
     def setUp(self):
-        utils.HASH_PATH_SUFFIX = 'endcap'
+        ondisk.HASH_PATH_SUFFIX = 'endcap'
-        utils.HASH_PATH_PREFIX = ''
+        ondisk.HASH_PATH_PREFIX = ''
         # Setup a test ring (stolen from common/test_ring.py)
         self.testdir = tempfile.mkdtemp()
         self.devices = os.path.join(self.testdir, 'node')

test/unit/obj/test_server.py

@@ -32,9 +32,10 @@ from test.unit import FakeLogger
 from test.unit import connect_tcp, readuntil2crlfs
 from swift.obj import server as object_server
 from swift.obj import diskfile
-from swift.common import utils
+from swift.common.utils import mkdirs, NullLogger, public, replication
-from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
+from swift.common.ondisk import hash_path, normalize_timestamp, \
-    NullLogger, storage_directory, public, replication
+    storage_directory
+from swift.common import ondisk
 from swift.common import constraints
 from eventlet import tpool
 from swift.common.swob import Request, HeaderKeyDict
@@ -45,8 +46,8 @@ class TestObjectController(unittest.TestCase):
 
     def setUp(self):
         """Set up for testing swift.object.server.ObjectController"""
-        utils.HASH_PATH_SUFFIX = 'endcap'
+        ondisk.HASH_PATH_SUFFIX = 'endcap'
-        utils.HASH_PATH_PREFIX = 'startcap'
+        ondisk.HASH_PATH_PREFIX = 'startcap'
         self.testdir = \
             os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController')
         mkdirs(os.path.join(self.testdir, 'sda1', 'tmp'))
@@ -1904,8 +1905,8 @@ class TestObjectController(unittest.TestCase):
                  'x-trans-id': '-'})})
 
     def test_async_update_saves_on_exception(self):
-        _prefix = utils.HASH_PATH_PREFIX
+        _prefix = ondisk.HASH_PATH_PREFIX
-        utils.HASH_PATH_PREFIX = ''
+        ondisk.HASH_PATH_PREFIX = ''
 
         def fake_http_connect(*args):
             raise Exception('test')
@@ -1918,7 +1919,7 @@ class TestObjectController(unittest.TestCase):
                 {'x-timestamp': '1', 'x-out': 'set'}, 'sda1')
         finally:
             object_server.http_connect = orig_http_connect
-            utils.HASH_PATH_PREFIX = _prefix
+            ondisk.HASH_PATH_PREFIX = _prefix
         self.assertEquals(
             pickle.load(open(os.path.join(
                 self.testdir, 'sda1', 'async_pending', 'a83',
@@ -1928,8 +1929,8 @@ class TestObjectController(unittest.TestCase):
              'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'})
 
     def test_async_update_saves_on_non_2xx(self):
-        _prefix = utils.HASH_PATH_PREFIX
+        _prefix = ondisk.HASH_PATH_PREFIX
-        utils.HASH_PATH_PREFIX = ''
+        ondisk.HASH_PATH_PREFIX = ''
 
         def fake_http_connect(status):
 
@@ -1963,7 +1964,7 @@ class TestObjectController(unittest.TestCase):
                      'op': 'PUT'})
         finally:
             object_server.http_connect = orig_http_connect
-            utils.HASH_PATH_PREFIX = _prefix
+            ondisk.HASH_PATH_PREFIX = _prefix
 
     def test_async_update_does_not_save_on_2xx(self):
 

test/unit/obj/test_updater.py

@@ -27,17 +27,17 @@ from eventlet import spawn, Timeout, listen
 from swift.obj import updater as object_updater, server as object_server
 from swift.obj.server import ASYNCDIR
 from swift.common.ring import RingData
-from swift.common import utils
+from swift.common.utils import mkdirs, write_pickle
-from swift.common.utils import hash_path, normalize_timestamp, mkdirs, \
+from swift.common import ondisk
-    write_pickle
+from swift.common.ondisk import hash_path, normalize_timestamp
 from test.unit import FakeLogger
 
 
 class TestObjectUpdater(unittest.TestCase):
 
     def setUp(self):
-        utils.HASH_PATH_SUFFIX = 'endcap'
+        ondisk.HASH_PATH_SUFFIX = 'endcap'
-        utils.HASH_PATH_PREFIX = ''
+        ondisk.HASH_PATH_PREFIX = ''
         self.testdir = os.path.join(os.path.dirname(__file__),
                                     'object_updater')
         rmtree(self.testdir, ignore_errors=1)

test/unit/proxy/test_server.py

@@ -47,7 +47,9 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \
     MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
     MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH
 from swift.common import utils
-from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
+from swift.common.utils import mkdirs, NullLogger
+from swift.common import ondisk
+from swift.common.ondisk import normalize_timestamp
 from swift.common.wsgi import monkey_patch_mimetools
 from swift.proxy.controllers.obj import SegmentedIterable
 from swift.proxy.controllers.base import get_container_memcache_key, \
@@ -73,7 +75,7 @@ def request_init(self, *args, **kwargs):
 
 
 def setup():
-    utils.HASH_PATH_SUFFIX = 'endcap'
+    ondisk.HASH_PATH_SUFFIX = 'endcap'
     global _testdir, _test_servers, _test_sockets, \
         _orig_container_listing_limit, _test_coros, _orig_SysLogHandler
     _orig_SysLogHandler = utils.SysLogHandler